diff --git a/cmd/btfs/daemon.go b/cmd/btfs/daemon.go index d5dc0790e..a220315a3 100644 --- a/cmd/btfs/daemon.go +++ b/cmd/btfs/daemon.go @@ -717,6 +717,9 @@ If the user need to start multiple nodes on the same machine, the configuration functest(cfg.Services.OnlineServerDomain, cfg.Identity.PeerID, hValue) } + // init s3 providers + s3.InitProviders(statestore) + // access-key init accesskey.InitService(s3.GetProviders()) diff --git a/core/commands/files.go b/core/commands/files.go index 950132efe..ae9272310 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -735,7 +735,7 @@ stat' on the file or any of its ancestors. }, Arguments: []cmds.Argument{ cmds.StringArg("path", true, false, "Path to write to."), - cmds.FileArg("data", true, false, "Data to write.").EnableStdin(), + cmds.FileArg("data", true, false, "data to write.").EnableStdin(), }, Options: []cmds.Option{ cmds.Int64Option(filesOffsetOptionName, "o", "Byte offset to begin writing at."), diff --git a/core/commands/object/object.go b/core/commands/object/object.go index 524e490f0..3a9d4c28b 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -270,7 +270,7 @@ Supported values are: Type: Node{}, Encoders: cmds.EncoderMap{ cmds.Protobuf: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *Node) error { - // deserialize the Data field as text as this was the standard behaviour + // deserialize the data field as text as this was the standard behaviour object, err := deserializeNode(out, "text") if err != nil { return nil @@ -371,20 +371,20 @@ It reads from stdin, and the output is a base58 encoded multihash. 'btfs object put' is a plumbing command for storing DAG nodes. It reads from stdin, and the output is a base58 encoded multihash. -Data should be in the format specified by the --inputenc flag. +data should be in the format specified by the --inputenc flag. --inputenc may be one of the following: * "protobuf" * "json" (default) Examples: - $ echo '{ "Data": "abc" }' | btfs object put + $ echo '{ "data": "abc" }' | btfs object put This creates a node with the data 'abc' and no links. For an object with links, create a file named 'node.json' with the contents: { - "Data": "another", + "data": "another", "Links": [ { "Name": "some link", "Hash": "QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V", @@ -399,7 +399,7 @@ And then run: }, Arguments: []cmds.Argument{ - cmds.FileArg("data", true, false, "Data to be stored as a DAG object.").EnableStdin(), + cmds.FileArg("data", true, false, "data to be stored as a DAG object.").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(inputencOptionName, "Encoding type of input data. One of: {\"protobuf\", \"json\"}.").WithDefault("json"), diff --git a/core/commands/object/patch.go b/core/commands/object/patch.go index f2eb0dc4e..ff2c6933e 100644 --- a/core/commands/object/patch.go +++ b/core/commands/object/patch.go @@ -46,7 +46,7 @@ the limit will not be respected by the network. }, Arguments: []cmds.Argument{ cmds.StringArg("root", true, false, "The hash of the node to modify."), - cmds.FileArg("data", true, false, "Data to append.").EnableStdin(), + cmds.FileArg("data", true, false, "data to append.").EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index 276f6a0dc..9ac110d60 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -66,10 +66,10 @@ func verifyFile(t *testing.T, path string, wantData []byte) { t.Fatal(err) } if len(isData) != len(wantData) { - t.Fatal("Data not equal - length check failed") + t.Fatal("data not equal - length check failed") } if !bytes.Equal(isData, wantData) { - t.Fatal("Data not equal") + t.Fatal("data not equal") } } @@ -328,7 +328,7 @@ func TestAppendFile(t *testing.T) { t.Fatal(err) } if !bytes.Equal(rbuf, data) { - t.Fatal("Data inconsistent!") + t.Fatal("data inconsistent!") } } @@ -458,7 +458,7 @@ func TestFSThrash(t *testing.T) { } if !bytes.Equal(data, out) { - t.Errorf("Data didn't match in %s: expected %v, got %v", name, data, out) + t.Errorf("data didn't match in %s: expected %v, got %v", name, data, out) } } } diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 7e92aa6bf..dc7451d42 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -272,7 +272,7 @@ func (s *Node) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadR if err != nil { return err } - // Data has a capacity of Size + // data has a capacity of Size buf := resp.Data[:int(req.Size)] n, err := io.ReadFull(r, buf) resp.Data = buf[:n] diff --git a/s3/consts/consts.go b/s3/consts/consts.go index 09c388843..53c163e49 100644 --- a/s3/consts/consts.go +++ b/s3/consts/consts.go @@ -28,10 +28,11 @@ const ( AssumeRole = "AssumeRole" SignV4Algorithm = "AWS4-HMAC-SHA256" - DefaultLocation = "us-east-1" - DefaultBucketACL = s3.BucketCannedACLPublicRead - DefaultObjectACL = "" - AllUsersURI = "http://acs.amazonaws.com/groups/global/AllUsers" + DefaultServerInfo = "BTFS" + DefaultLocation = "us-east-1" + DefaultBucketACL = s3.BucketCannedACLPublicRead + DefaultObjectACL = "" + AllUsersURI = "http://acs.amazonaws.com/groups/global/AllUsers" ) var SupportedLocations = map[string]bool{ diff --git a/s3/handlers/handlers.go b/s3/handlers/handlers.go index f47c7dcdf..bd82b4349 100644 --- a/s3/handlers/handlers.go +++ b/s3/handlers/handlers.go @@ -12,8 +12,6 @@ import ( "strconv" ) -const lockPrefix = "s3:lock/" - var _ Handlerser = (*Handlers)(nil) type Handlers struct { diff --git a/s3/handlers/handlers_bucket.go b/s3/handlers/handlers_bucket.go index ddf26b4a1..e0c246b96 100644 --- a/s3/handlers/handlers_bucket.go +++ b/s3/handlers/handlers_bucket.go @@ -44,7 +44,7 @@ func (h *Handlers) CreateBucketHandler(w http.ResponseWriter, r *http.Request) { return } - responses.WritePutBucketResponse(w, r) + responses.WriteCreateBucketResponse(w, r) return } @@ -154,7 +154,7 @@ func (h *Handlers) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) { }() req, rerr := requests.ParsePutBucketAclRequest(r) - if err != nil { + if rerr != nil { err = rerr responses.WriteErrorResponse(w, r, rerr) return diff --git a/s3/handlers/handlers_utils.go b/s3/handlers/utils.go similarity index 100% rename from s3/handlers/handlers_utils.go rename to s3/handlers/utils.go diff --git a/s3/requests/parsers.go b/s3/requests/parsers.go index e0ac814a6..09175e04f 100644 --- a/s3/requests/parsers.go +++ b/s3/requests/parsers.go @@ -1,12 +1,9 @@ package requests import ( - "errors" - "fmt" "github.com/bittorrent/go-btfs/s3/cctx" "github.com/bittorrent/go-btfs/s3/responses" "net/http" - "reflect" ) // CreateBucketRequest . @@ -17,23 +14,6 @@ type CreateBucketRequest struct { Region string } -// todo: parse aws request use aws struct -func ParseS3Request(r *http.Request, v interface{}) (err error) { - rv := reflect.ValueOf(v) - if rv.Kind() != reflect.Pointer || rv.IsNil() { - err = errors.New("invalid value must be non nil pointer") - return - } - - rt := reflect.TypeOf(v).Elem() - n := rt.NumField() - for i := 0; i < n; i++ { - f := rt.Field(i) - fmt.Println(f) - } - return -} - func ParseCreateBucketRequest(r *http.Request) (req *CreateBucketRequest, rerr *responses.Error) { req = &CreateBucketRequest{} req.AccessKey = cctx.GetAccessKey(r) diff --git a/s3/requests/parsers_common.go b/s3/requests/parsers_common.go index 488da5791..8bdf75b65 100644 --- a/s3/requests/parsers_common.go +++ b/s3/requests/parsers_common.go @@ -14,7 +14,8 @@ import ( ) func parseBucket(r *http.Request) (bucket string, rerr *responses.Error) { - err := s3utils.CheckValidBucketNameStrict(mux.Vars(r)["bucket"]) + bucket = mux.Vars(r)["bucket"] + err := s3utils.CheckValidBucketNameStrict(bucket) if err != nil { rerr = responses.ErrInvalidBucketName } diff --git a/s3/requests/types_common.go b/s3/requests/types_common.go deleted file mode 100644 index e2107f405..000000000 --- a/s3/requests/types_common.go +++ /dev/null @@ -1,10 +0,0 @@ -package requests - -import "encoding/xml" - -// createBucketConfiguration container for bucket configuration request from client. -// Used for parsing the location from the request body for Makebucket. -type createBucketLocationConfiguration struct { - XMLName xml.Name `xml:"CreateBucketConfiguration" json:"-"` - Location string `xml:"LocationConstraint"` -} diff --git a/s3/responses/object_header.go b/s3/responses/object_header.go deleted file mode 100644 index 2f66c84fe..000000000 --- a/s3/responses/object_header.go +++ /dev/null @@ -1,64 +0,0 @@ -package responses - -import ( - "github.com/bittorrent/go-btfs/s3/consts" - "github.com/bittorrent/go-btfs/s3/services/object" - "net/http" - "net/url" - "strconv" - "strings" -) - -// SetObjectHeaders Write object header -func SetObjectHeaders(w http.ResponseWriter, r *http.Request, objInfo object.Object) { - // set common headers - setCommonHeaders(w, r) - - // Set last modified time. - lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat) - w.Header().Set(consts.LastModified, lastModified) - - // Set Etag if available. - if objInfo.ETag != "" { - w.Header()[consts.ETag] = []string{"\"" + objInfo.ETag + "\""} - } - - if objInfo.ContentType != "" { - w.Header().Set(consts.ContentType, objInfo.ContentType) - } - - if objInfo.ContentEncoding != "" { - w.Header().Set(consts.ContentEncoding, objInfo.ContentEncoding) - } - - if !objInfo.Expires.IsZero() { - w.Header().Set(consts.Expires, objInfo.Expires.UTC().Format(http.TimeFormat)) - } - - // Set content length - w.Header().Set(consts.ContentLength, strconv.FormatInt(objInfo.Size, 10)) - - // Set the relevant version ID as part of the response header. - if objInfo.VersionID != "" { - w.Header()[consts.AmzVersionID] = []string{objInfo.VersionID} - } - -} - -// SetHeadGetRespHeaders - set any requested parameters as response headers. -func SetHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { - for k, v := range reqParams { - if header, ok := supportedHeadGetReqParams[strings.ToLower(k)]; ok { - w.Header()[header] = v - } - } -} - -// supportedHeadGetReqParams - supported request parameters for GET and HEAD presigned request. -var supportedHeadGetReqParams = map[string]string{ - "response-expires": consts.Expires, - "response-content-type": consts.ContentType, - "response-content-encoding": consts.ContentEncoding, - "response-content-language": consts.ContentLanguage, - "response-content-disposition": consts.ContentDisposition, -} diff --git a/s3/responses/response.go b/s3/responses/response.go deleted file mode 100644 index 04e2cedb4..000000000 --- a/s3/responses/response.go +++ /dev/null @@ -1,228 +0,0 @@ -package responses - -import ( - "bytes" - "encoding/json" - "encoding/xml" - "errors" - "fmt" - "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/bittorrent/go-btfs/s3/consts" - "github.com/gorilla/mux" - logging "github.com/ipfs/go-log/v2" - "net/http" - "net/url" - "path" - "strconv" - "time" -) - -var log = logging.Logger("resp") - -type mimeType string - -const ( - mimeNone mimeType = "" - mimeJSON mimeType = "application/json" - //mimeXML application/xml UTF-8 - mimeXML mimeType = " application/xml" -) - -func owner(accessKey string) *s3.Owner { - return new(s3.Owner).SetID(accessKey).SetDisplayName(accessKey) -} - -func ownerFullControlGrant(accessKey string) *s3.Grant { - return new(s3.Grant).SetGrantee(new(s3.Grantee).SetType(s3.TypeCanonicalUser).SetID(accessKey).SetDisplayName(accessKey)).SetPermission(s3.PermissionFullControl) -} - -var ( - allUsersReadGrant = new(s3.Grant).SetGrantee(new(s3.Grantee).SetType(s3.TypeGroup).SetURI(consts.AllUsersURI)).SetPermission(s3.PermissionRead) - allUsersWriteGrant = new(s3.Grant).SetGrantee(new(s3.Grantee).SetType(s3.TypeGroup).SetURI(consts.AllUsersURI)).SetPermission(s3.PermissionWrite) -) - -// APIErrorResponse - error response format -type APIErrorResponse struct { - XMLName xml.Name `xml:"Error" json:"-"` - Code string - Message string - Resource string - RequestID string `xml:"RequestId" json:"RequestId"` - HostID string `xml:"HostId" json:"HostId"` -} - -type RESTErrorResponse struct { - XMLName xml.Name `xml:"Error" json:"-"` - Code string `xml:"Code" json:"Code"` - Message string `xml:"Message" json:"Message"` - Resource string `xml:"Resource" json:"Resource"` - RequestID string `xml:"RequestId" json:"RequestId"` - Key string `xml:"Key,omitempty" json:"Key,omitempty"` - BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"` -} - -func getRESTErrorResponse(err *Error, resource string, bucket, object string) RESTErrorResponse { - return RESTErrorResponse{ - Code: err.Code(), - BucketName: bucket, - Key: object, - Message: err.Description(), - Resource: resource, - RequestID: fmt.Sprintf("%d", time.Now().UnixNano()), - } -} - -func WriteErrorResponseHeadersOnly(w http.ResponseWriter, r *http.Request, err error) { - var rerr *Error - if !errors.As(err, &rerr) { - rerr = ErrInternalError - } - writeResponse(w, r, rerr.HTTPStatusCode(), nil, mimeNone) -} - -// WriteErrorResponse write ErrorResponse -func WriteErrorResponse(w http.ResponseWriter, r *http.Request, rerr *Error) { - errorResponse := RESTErrorResponse{ - Code: rerr.Code(), - BucketName: mux.Vars(r)["bucket"], - Key: mux.Vars(r)["object"], - Message: rerr.Description(), - Resource: r.URL.Path, - RequestID: fmt.Sprintf("%d", time.Now().UnixNano()), - } - WriteXMLResponse(w, r, rerr.HTTPStatusCode(), errorResponse) -} - -// WriteSuccessResponseHeadersOnly write SuccessResponseHeadersOnly -func WriteSuccessResponseHeadersOnly(w http.ResponseWriter, r *http.Request) { - writeResponse(w, r, http.StatusOK, nil, mimeNone) -} - -// WriteSuccessResponse write SuccessResponseHeadersOnly -func WriteSuccessResponse(w http.ResponseWriter, r *http.Request) { - writeResponse(w, r, http.StatusOK, nil, mimeNone) -} - -// WriteSuccessResponseXML Write Success Response XML -func WriteSuccessResponseXML(w http.ResponseWriter, r *http.Request, response interface{}) { - WriteXMLResponse(w, r, http.StatusOK, response) -} - -// WriteXMLResponse Write XMLResponse -func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) { - writeResponse(w, r, statusCode, encodeXMLResponse(response), mimeXML) -} - -func writeResponse(w http.ResponseWriter, r *http.Request, statusCode int, response []byte, mType mimeType) { - setCommonHeaders(w, r) - if response != nil { - w.Header().Set(consts.ContentLength, strconv.Itoa(len(response))) - } - if mType != mimeNone { - w.Header().Set(consts.ContentType, string(mType)) - } - w.WriteHeader(statusCode) - if response != nil { - _, err := w.Write(response) - if err != nil { - log.Errorf("write err: %v", err) - } - w.(http.Flusher).Flush() - } -} - -func setCommonHeaders(w http.ResponseWriter, r *http.Request) { - w.Header().Set(consts.ServerInfo, "FDS") - w.Header().Set(consts.AmzRequestID, fmt.Sprintf("%d", time.Now().UnixNano())) - w.Header().Set(consts.AcceptRanges, "bytes") - if r.Header.Get("Origin") != "" { - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Credentials", "true") - } -} - -// encodeXMLResponse Encodes the response headers into XML format. -func encodeXMLResponse(response interface{}) []byte { - var buf bytes.Buffer - buf.WriteString(xml.Header) - err := xmlutil.BuildXML(response, xml.NewEncoder(&buf)) - if err != nil { - panic(err) - } - bs := buf.Bytes() - fmt.Println(string(bs)) - return bs -} - -// WriteErrorResponseJSON - writes error response in JSON format; -// useful for admin APIs. -func WriteErrorResponseJSON(w http.ResponseWriter, err error, reqURL *url.URL, host string) { - var rerr *Error - if !errors.As(err, &rerr) { - rerr = ErrInternalError - } - // Generate error response. - errorResponse := getAPIErrorResponse(rerr, reqURL.Path, w.Header().Get(consts.AmzRequestID), host) - encodedErrorResponse := encodeResponseJSON(errorResponse) - writeResponseSimple(w, rerr.HTTPStatusCode(), encodedErrorResponse, mimeJSON) -} - -// getErrorResponse gets in standard error and resource value and -// provides a encodable populated response values -func getAPIErrorResponse(err *Error, resource, requestID, hostID string) APIErrorResponse { - return APIErrorResponse{ - Code: err.Code(), - Message: err.Description(), - Resource: resource, - RequestID: requestID, - HostID: hostID, - } -} - -// Encodes the response headers into JSON format. -func encodeResponseJSON(response interface{}) []byte { - var bytesBuffer bytes.Buffer - e := json.NewEncoder(&bytesBuffer) - e.Encode(response) - return bytesBuffer.Bytes() -} - -// WriteSuccessResponseJSON writes success headers and response if any, -// with content-type set to `application/json`. -func WriteSuccessResponseJSON(w http.ResponseWriter, response []byte) { - writeResponseSimple(w, http.StatusOK, response, mimeJSON) -} - -func writeResponseSimple(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { - if mType != mimeNone { - w.Header().Set(consts.ContentType, string(mType)) - } - w.Header().Set(consts.ContentLength, strconv.Itoa(len(response))) - w.WriteHeader(statusCode) - if response != nil { - w.Write(response) - } -} - -// WriteSuccessNoContent writes success headers with http status 204 -func WriteSuccessNoContent(w http.ResponseWriter) { - writeResponseSimple(w, http.StatusNoContent, nil, mimeNone) -} - -func setPutObjHeaders(w http.ResponseWriter, etag, cid string, delete bool) { - if etag != "" && !delete { - w.Header()[consts.ETag] = []string{`"` + etag + `"`} - } - if cid != "" { - w.Header()[consts.CID] = []string{cid} - } -} - -func pathClean(p string) string { - cp := path.Clean(p) - if cp == "." { - return "" - } - return cp -} diff --git a/s3/responses/response_multipart.go b/s3/responses/response_multipart.go deleted file mode 100644 index 62188e8ab..000000000 --- a/s3/responses/response_multipart.go +++ /dev/null @@ -1,26 +0,0 @@ -package responses - -import ( - "github.com/bittorrent/go-btfs/s3/services/object" - "net/http" -) - -func WriteCreateMultipartUploadResponse(w http.ResponseWriter, r *http.Request, bucname, objname, uploadID string) { - resp := GenerateInitiateMultipartUploadResponse(bucname, objname, uploadID) - WriteSuccessResponseXML(w, r, resp) -} - -func WriteAbortMultipartUploadResponse(w http.ResponseWriter, r *http.Request) { - WriteSuccessNoContent(w) -} - -func WriteUploadPartResponse(w http.ResponseWriter, r *http.Request, part object.Part) { - setPutObjHeaders(w, part.ETag, part.CID, false) - WriteSuccessResponseHeadersOnly(w, r) -} - -func WriteCompleteMultipartUploadResponse(w http.ResponseWriter, r *http.Request, bucname, objname, region string, obj object.Object) { - resp := GenerateCompleteMultipartUploadResponse(bucname, objname, region, obj) - setPutObjHeaders(w, obj.ETag, obj.CID, false) - WriteSuccessResponseXML(w, r, resp) -} diff --git a/s3/responses/responses.go b/s3/responses/responses.go new file mode 100644 index 000000000..f9f2ac782 --- /dev/null +++ b/s3/responses/responses.go @@ -0,0 +1,364 @@ +package responses + +import ( + "bytes" + "encoding/base64" + "encoding/xml" + "errors" + "fmt" + "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" + "github.com/bittorrent/go-btfs/s3/consts" + "io" + "math" + "net/http" + "reflect" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/private/protocol" +) + +const ( + mimeTypeXml = "application/xml" + noPayload = "nopayload" +) + +const ( + floatNaN = "NaN" + floatInf = "Infinity" + floatNegInf = "-Infinity" +) + +var errValueNotSet = fmt.Errorf("value not set") + +var byteSliceType = reflect.TypeOf([]byte{}) + +func WriteResponse(w http.ResponseWriter, statusCode int, output interface{}, locationName string) (err error) { + if locationName != "" { + output = wrapOutput(output, locationName) + } + + defer func() { + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + }() + + if !outputFilled(output) { + w.WriteHeader(statusCode) + return + } + + typ := getPayloadType(output) + + if typ == noPayload { + err = buildHeader(w.Header(), output) + if err != nil { + return + } + w.WriteHeader(statusCode) + return + } + + var ( + body io.ReadCloser + contentLength int + contentType string + ) + defer func() { + if body != nil { + _ = body.Close() + } + }() + + switch typ { + case "structure", "": + body, contentLength, contentType, err = buildXMLBody(output) + default: + body, contentLength, contentType, err = buildRESTBody(output) + } + if err != nil { + return + } + + if contentLength != -1 { + w.Header().Set(consts.ContentLength, fmt.Sprintf("%d", contentLength)) + } + + if contentType != "" { + w.Header().Set(consts.ContentType, contentType) + } + + err = buildHeader(w.Header(), output) + if err != nil { + return + } + + w.WriteHeader(statusCode) + + if body != nil { + _, err = io.Copy(w, body) + } + + return +} + +func wrapOutput(v interface{}, locationName string) (wrapper interface{}) { + outputTag := fmt.Sprintf(`locationName:"%s" type:"structure"`, locationName) + fields := []reflect.StructField{ + { + Name: "_", + Type: reflect.TypeOf(struct{}{}), + Tag: `payload:"Output" type:"structure"`, + PkgPath: "responses", + }, + { + Name: "Output", + Type: reflect.TypeOf(v), + Tag: reflect.StructTag(outputTag), + }, + } + wrapperTyp := reflect.StructOf(fields) + wrapperVal := reflect.New(wrapperTyp) + wrapperVal.Elem().Field(1).Set(reflect.ValueOf(v)) + wrapper = wrapperVal.Interface() + return +} + +func outputFilled(output interface{}) bool { + return reflect.Indirect(reflect.ValueOf(output)).IsValid() +} + +func getPayloadType(output interface{}) (typ string) { + typ = noPayload + v := reflect.Indirect(reflect.ValueOf(output)) + if !v.IsValid() { + return + } + field, ok := v.Type().FieldByName("_") + if !ok { + return + } + noPayloadValue := field.Tag.Get(noPayload) + if noPayloadValue != "" { + return + } + payloadName := field.Tag.Get("payload") + if payloadName == "" { + return + } + member, ok := v.Type().FieldByName(payloadName) + if !ok { + return + } + typ = member.Tag.Get("type") + return +} + +func buildXMLBody(output interface{}) (body io.ReadCloser, contentLength int, contentType string, err error) { + var buf bytes.Buffer + buf.WriteString(xml.Header) + err = xmlutil.BuildXML(output, xml.NewEncoder(&buf)) + if err != nil { + return + } + body = io.NopCloser(&buf) + contentLength = buf.Len() + contentType = mimeTypeXml + return +} + +func buildRESTBody(output interface{}) (body io.ReadCloser, contentLength int, contentType string, err error) { + v := reflect.Indirect(reflect.ValueOf(output)) + field, _ := v.Type().FieldByName("_") + payloadName := field.Tag.Get("payload") + payload := reflect.Indirect(v.FieldByName(payloadName)) + if !payload.IsValid() || payload.Interface() == nil { + return + } + switch pIface := payload.Interface().(type) { + case io.ReadCloser: + body = pIface + contentLength = -1 + case []byte: + body = io.NopCloser(bytes.NewBuffer(pIface)) + contentLength = len(pIface) + case string: + body = io.NopCloser(bytes.NewBufferString(pIface)) + contentLength = len(pIface) + default: + err = fmt.Errorf( + "unknown payload type %s", + payload.Type(), + ) + } + return +} + +func buildHeader(header http.Header, output interface{}) (err error) { + v := reflect.ValueOf(output).Elem() + for i := 0; i < v.NumField(); i++ { + ft := v.Type().Field(i) + fv := v.Field(i) + fk := fv.Kind() + + if !fv.IsValid() { + continue + } + + if n := ft.Name; n[0:1] == strings.ToLower(n[0:1]) { + continue + } + + if fk == reflect.Ptr { + fv = fv.Elem() + fk = fv.Kind() + if !fv.IsValid() { + continue + } + } else if fk == reflect.Interface { + if !fv.Elem().IsValid() { + continue + } + } + + if ft.Tag.Get("ignore") != "" { + continue + } + + if ft.Tag.Get("marshal-as") == "blob" { + fv = fv.Convert(byteSliceType) + } + + switch ft.Tag.Get("location") { + case "headers": + err = writeHeaderMap(&header, fv, ft.Tag) + case "header": + name := ft.Tag.Get("locationName") + if name == "" { + name = ft.Name + } + err = writeHeader(&header, fv, name, ft.Tag) + } + + if err != nil { + return + } + } + + return +} + +func writeHeader(header *http.Header, v reflect.Value, name string, tag reflect.StructTag) (err error) { + str, err := convertType(v, tag) + if errors.Is(err, errValueNotSet) { + err = nil + return + } + if err != nil { + return + } + name = strings.TrimSpace(name) + str = strings.TrimSpace(str) + header.Add(name, str) + return +} + +func writeHeaderMap(header *http.Header, v reflect.Value, tag reflect.StructTag) (err error) { + prefix := tag.Get("locationName") + for _, key := range v.MapKeys() { + var str string + str, err = convertType(v.MapIndex(key), tag) + if errors.Is(err, errValueNotSet) { + err = nil + continue + } + if err != nil { + return + } + keyStr := strings.TrimSpace(key.String()) + str = strings.TrimSpace(str) + header.Add(prefix+keyStr, str) + } + return +} + +func convertType(v reflect.Value, tag reflect.StructTag) (str string, err error) { + v = reflect.Indirect(v) + if !v.IsValid() { + err = errValueNotSet + return + } + + switch value := v.Interface().(type) { + case string: + if tag.Get("suppressedJSONValue") == "true" && tag.Get("location") == "header" { + value = base64.StdEncoding.EncodeToString([]byte(value)) + } + str = value + case []*string: + if tag.Get("location") != "header" || tag.Get("enum") == "" { + return "", fmt.Errorf("%T is only supported with location header and enum shapes", value) + } + if len(value) == 0 { + return "", errValueNotSet + } + + buff := &bytes.Buffer{} + for i, sv := range value { + if sv == nil || len(*sv) == 0 { + continue + } + if i != 0 { + buff.WriteRune(',') + } + item := *sv + if strings.Index(item, `,`) != -1 || strings.Index(item, `"`) != -1 { + item = strconv.Quote(item) + } + buff.WriteString(item) + } + str = string(buff.Bytes()) + case []byte: + str = base64.StdEncoding.EncodeToString(value) + case bool: + str = strconv.FormatBool(value) + case int64: + str = strconv.FormatInt(value, 10) + case float64: + switch { + case math.IsNaN(value): + str = floatNaN + case math.IsInf(value, 1): + str = floatInf + case math.IsInf(value, -1): + str = floatNegInf + default: + str = strconv.FormatFloat(value, 'f', -1, 64) + } + case time.Time: + format := tag.Get("timestampFormat") + if len(format) == 0 { + format = protocol.RFC822TimeFormatName + if tag.Get("location") == "querystring" { + format = protocol.ISO8601TimeFormatName + } + } + str = protocol.FormatTime(format, value) + case aws.JSONValue: + if len(value) == 0 { + return "", errValueNotSet + } + escaping := protocol.NoEscape + if tag.Get("location") == "header" { + escaping = protocol.Base64Escape + } + str, err = protocol.EncodeJSONValue(value, escaping) + default: + err = fmt.Errorf("unsupported value for param %v (%s)", v.Interface(), v.Type()) + } + + return +} diff --git a/s3/responses/response_bucket.go b/s3/responses/responses_bucket.go similarity index 57% rename from s3/responses/response_bucket.go rename to s3/responses/responses_bucket.go index 9bced083f..8430c284e 100644 --- a/s3/responses/response_bucket.go +++ b/s3/responses/responses_bucket.go @@ -2,58 +2,51 @@ package responses import ( "github.com/aws/aws-sdk-go/service/s3" - "github.com/bittorrent/go-btfs/s3/consts" "github.com/bittorrent/go-btfs/s3/services/object" "net/http" ) -func WritePutBucketResponse(w http.ResponseWriter, r *http.Request) { - if cp := pathClean(r.URL.Path); cp != "" { - w.Header().Set(consts.Location, cp) - } - WriteSuccessResponse(w, r) +func WriteCreateBucketResponse(w http.ResponseWriter, r *http.Request) { + output := new(s3.CreateBucketOutput).SetLocation(pathClean(r.URL.Path)) + WriteSuccessResponse(w, output, "") return } func WriteHeadBucketResponse(w http.ResponseWriter, r *http.Request) { - WriteSuccessResponse(w, r) + output := new(s3.HeadBucketOutput) + WriteSuccessResponse(w, output, "") return } + func WriteDeleteBucketResponse(w http.ResponseWriter) { - WriteSuccessNoContent(w) + output := new(s3.DeleteBucketOutput) + _ = WriteResponse(w, http.StatusOK, output, "") return } -type ListBucketResponse struct { - ListAllMyBucketsResult s3.ListBucketsOutput `xml:"ListAllMyBucketsResult"` -} - func WriteListBucketsResponse(w http.ResponseWriter, r *http.Request, accessKey string, buckets []*object.Bucket) { - resp := &ListBucketResponse{} - resp.ListAllMyBucketsResult.SetOwner(owner(accessKey)) + output := new(s3.ListBucketsOutput) + output.SetOwner(owner(accessKey)) s3Buckets := make([]*s3.Bucket, 0) for _, buc := range buckets { s3Bucket := new(s3.Bucket).SetName(buc.Name).SetCreationDate(buc.Created) s3Buckets = append(s3Buckets, s3Bucket) } - resp.ListAllMyBucketsResult.SetBuckets(s3Buckets) - WriteSuccessResponseXML(w, r, resp) + output.SetBuckets(s3Buckets) + WriteSuccessResponse(w, output, "ListAllMyBucketsResult") return } func WritePutBucketAclResponse(w http.ResponseWriter, r *http.Request) { - WriteSuccessResponse(w, r) + output := new(s3.PutBucketAclOutput) + WriteSuccessResponse(w, output, "") return } -type GetBucketACLResponse struct { - AccessControlPolicy s3.GetBucketAclOutput `xml:"AccessControlPolicy"` -} - func WriteGetBucketACLResponse(w http.ResponseWriter, r *http.Request, accessKey string, acl string) { - resp := GetBucketACLResponse{} - resp.AccessControlPolicy.SetOwner(owner(accessKey)) + output := new(s3.GetBucketAclOutput) + output.SetOwner(owner(accessKey)) grants := make([]*s3.Grant, 0) grants = append(grants, ownerFullControlGrant(accessKey)) switch acl { @@ -65,7 +58,7 @@ func WriteGetBucketACLResponse(w http.ResponseWriter, r *http.Request, accessKey default: panic("unknown acl") } - resp.AccessControlPolicy.SetGrants(grants) - WriteSuccessResponseXML(w, r, resp) + output.SetGrants(grants) + WriteSuccessResponse(w, output, "AccessControlPolicy") return } diff --git a/s3/responses/responses_common.go b/s3/responses/responses_common.go new file mode 100644 index 000000000..7bc1864a4 --- /dev/null +++ b/s3/responses/responses_common.go @@ -0,0 +1,81 @@ +package responses + +import ( + "fmt" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/bittorrent/go-btfs/s3/consts" + "net/http" + "path" + "time" +) + +func owner(accessKey string) *s3.Owner { + return new(s3.Owner).SetID(accessKey).SetDisplayName(accessKey) +} + +func ownerFullControlGrant(accessKey string) *s3.Grant { + return new(s3.Grant).SetGrantee(new(s3.Grantee).SetType(s3.TypeCanonicalUser).SetID(accessKey).SetDisplayName(accessKey)).SetPermission(s3.PermissionFullControl) +} + +var ( + allUsersReadGrant = new(s3.Grant).SetGrantee(new(s3.Grantee).SetType(s3.TypeGroup).SetURI(consts.AllUsersURI)).SetPermission(s3.PermissionRead) + allUsersWriteGrant = new(s3.Grant).SetGrantee(new(s3.Grantee).SetType(s3.TypeGroup).SetURI(consts.AllUsersURI)).SetPermission(s3.PermissionWrite) +) + +func getRequestID() string { + return fmt.Sprintf("%d", time.Now().UnixNano()) +} + +func setCommonHeader(w http.ResponseWriter, requestId string) { + w.Header().Set(consts.ServerInfo, consts.DefaultServerInfo) + w.Header().Set(consts.AmzRequestID, requestId) + w.Header().Set(consts.AcceptRanges, "bytes") +} + +type ErrorOutput struct { + _ struct{} `type:"structure"` + Code string `locationName:"Code" type:"string"` + Message string `locationName:"Message" type:"string"` + Resource string `locationName:"Resource" type:"string"` + RequestID string `locationName:"RequestID" type:"string"` +} + +func WriteErrorResponse(w http.ResponseWriter, r *http.Request, rerr *Error) { + reqID := getRequestID() + setCommonHeader(w, reqID) + output := &ErrorOutput{ + Code: rerr.Code(), + Message: rerr.Description(), + Resource: pathClean(r.URL.Path), + RequestID: reqID, + } + err := WriteResponse(w, rerr.HTTPStatusCode(), output, "Error") + if err != nil { + fmt.Println("write response: ", err) + } +} + +func WriteSuccessResponse(w http.ResponseWriter, output interface{}, locationName string) { + setCommonHeader(w, getRequestID()) + err := WriteResponse(w, http.StatusOK, output, locationName) + if err != nil { + fmt.Println("write response: ", err) + } +} + +func setPutObjHeaders(w http.ResponseWriter, etag, cid string, delete bool) { + if etag != "" && !delete { + w.Header()[consts.ETag] = []string{`"` + etag + `"`} + } + if cid != "" { + w.Header()[consts.CID] = []string{cid} + } +} + +func pathClean(p string) string { + cp := path.Clean(p) + if cp == "." { + return "" + } + return cp +} diff --git a/s3/responses/responses_multipart.go b/s3/responses/responses_multipart.go new file mode 100644 index 000000000..fd7aff5d2 --- /dev/null +++ b/s3/responses/responses_multipart.go @@ -0,0 +1,21 @@ +package responses + +//func WriteCreateMultipartUploadResponse(w http.ResponseWriter, r *http.Request, bucname, objname, uploadID string) { +// resp := GenerateInitiateMultipartUploadResponse(bucname, objname, uploadID) +// WriteSuccessResponse(w, resp, "") +//} +// +//func WriteAbortMultipartUploadResponse(w http.ResponseWriter, r *http.Request) { +// WriteSuccessResponse(w, nil, "") +//} +// +//func WriteUploadPartResponse(w http.ResponseWriter, r *http.Request, part object.Part) { +// setPutObjHeaders(w, part.ETag, part.CID, false) +// WriteSuccessResponse(w, nil, "") +//} +// +//func WriteCompleteMultipartUploadResponse(w http.ResponseWriter, r *http.Request, bucname, objname, region string, obj object.Object) { +// resp := GenerateCompleteMultipartUploadResponse(bucname, objname, region, obj) +// setPutObjHeaders(w, obj.ETag, obj.CID, false) +// WriteSuccessResponse(w, resp, "") +//} diff --git a/s3/responses/response_object.go b/s3/responses/responses_object.go similarity index 85% rename from s3/responses/response_object.go rename to s3/responses/responses_object.go index 34bbf0073..edffad81d 100644 --- a/s3/responses/response_object.go +++ b/s3/responses/responses_object.go @@ -7,5 +7,5 @@ import ( func WritePutObjectResponse(w http.ResponseWriter, r *http.Request, obj object.Object) { setPutObjHeaders(w, obj.ETag, obj.CID, false) - WriteSuccessResponseHeadersOnly(w, r) + WriteSuccessResponse(w, nil, "") } diff --git a/s3/responses/types.go b/s3/responses/types.go deleted file mode 100644 index 55987629e..000000000 --- a/s3/responses/types.go +++ /dev/null @@ -1,310 +0,0 @@ -package responses - -import ( - "encoding/xml" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/bittorrent/go-btfs/s3/services/object" -) - -type AccessControlList struct { - Grant []*s3.Grant `xml:"Grant,omitempty"` -} - -type CanonicalUser struct { - ID string `xml:"ID"` - DisplayName string `xml:"DisplayName,omitempty"` -} - -// Grant grant -type Grant struct { - Grantee Grantee `xml:"Grantee"` - Permission Permission `xml:"Permission"` -} - -// Grantee grant -type Grantee struct { - XMLNS string `xml:"xmlns:xsi,attr"` - XMLXSI string `xml:"xsi:type,attr"` - Type string `xml:"Type"` - ID string `xml:"ID,omitempty"` - DisplayName string `xml:"DisplayName,omitempty"` - URI string `xml:"URI,omitempty"` -} - -// Permission May be one of READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL -type Permission string - -// ListAllMyBucketsResult List All Buckets Result -type ListAllMyBucketsResult struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult"` - Owner *s3.Owner - Buckets []*s3.Bucket `xml:"Buckets>Bucket"` -} - -type CopyObjectResponse struct { - CopyObjectResult CopyObjectResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult"` -} - -type CopyObjectResult struct { - LastModified string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` - ETag string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"` -} - -// LocationResponse - format for location response. -type LocationResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"` - Location string `xml:",chardata"` -} - -// ListObjectsResponse - format for list objects response. -type ListObjectsResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` - - Name string - Prefix string - Marker string - - // When response is truncated (the IsTruncated element value in the response - // is true), you can use the key name in this field as marker in the subsequent - // request to get next set of objects. Server lists objects in alphabetical - // order Note: This element is returned only if you have delimiter request parameter - // specified. If response does not include the NextMaker and it is truncated, - // you can use the value of the last Key in the response as the marker in the - // subsequent request to get the next set of object keys. - NextMarker string `xml:"NextMarker,omitempty"` - - MaxKeys int - Delimiter string - // A flag that indicates whether or not ListObjects returned all of the results - // that satisfied the search criteria. - IsTruncated bool - - Contents []Object - CommonPrefixes []CommonPrefix - - // Encoding type used to encode object keys in the response. - EncodingType string `xml:"EncodingType,omitempty"` -} - -// ListObjectsV2Response - format for list objects response. -type ListObjectsV2Response struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` - - Name string - Prefix string - StartAfter string `xml:"StartAfter,omitempty"` - // When response is truncated (the IsTruncated element value in the response - // is true), you can use the key name in this field as marker in the subsequent - // request to get next set of objects. Server lists objects in alphabetical - // order Note: This element is returned only if you have delimiter request parameter - // specified. If response does not include the NextMaker and it is truncated, - // you can use the value of the last Key in the response as the marker in the - // subsequent request to get the next set of object keys. - ContinuationToken string `xml:"ContinuationToken,omitempty"` - NextContinuationToken string `xml:"NextContinuationToken,omitempty"` - - KeyCount int - MaxKeys int - Delimiter string - // A flag that indicates whether or not ListObjects returned all of the results - // that satisfied the search criteria. - IsTruncated bool - - Contents []Object - CommonPrefixes []CommonPrefix - - // Encoding type used to encode object keys in the response. - EncodingType string `xml:"EncodingType,omitempty"` -} - -// Object container for object metadata -type Object struct { - Key string - LastModified string // time string of format "2006-01-02T15:04:05.000Z" - ETag string - CID string // CID - Size int64 - - // Owner of the object. - Owner s3.Owner - - // The class of storage used to store the object. - StorageClass string - - // UserMetadata user-defined metadata - UserMetadata StringMap `xml:"UserMetadata,omitempty"` -} - -// StringMap is a map[string]string -type StringMap map[string]string - -// MarshalXML - StringMap marshals into XML. -func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - tokens := []xml.Token{start} - - for key, value := range s { - t := xml.StartElement{} - t.Name = xml.Name{ - Space: "", - Local: key, - } - tokens = append(tokens, t, xml.CharData(value), xml.EndElement{Name: t.Name}) - } - - tokens = append(tokens, xml.EndElement{ - Name: start.Name, - }) - - for _, t := range tokens { - if err := e.EncodeToken(t); err != nil { - return err - } - } - - // flush to ensure tokens are written - return e.Flush() -} - -// CommonPrefix container for prefix response in ListObjectsResponse -type CommonPrefix struct { - Prefix string -} - -type InitiateMultipartUploadResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"` - - Bucket string - Key string - UploadID string `xml:"UploadId"` -} - -func GenerateInitiateMultipartUploadResponse(bucname, objname, uploadID string) InitiateMultipartUploadResponse { - return InitiateMultipartUploadResponse{ - Bucket: bucname, - Key: objname, - UploadID: uploadID, - } -} - -type CompleteMultipartUploadResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"` - - Location string - Bucket string - Key string - ETag string - - ChecksumCRC32 string - ChecksumCRC32C string - ChecksumSHA1 string - ChecksumSHA256 string -} - -func GenerateCompleteMultipartUploadResponse(bucname, objname, location string, obj object.Object) CompleteMultipartUploadResponse { - c := CompleteMultipartUploadResponse{ - Location: location, - Bucket: bucname, - Key: objname, - // AWS S3 quotes the ETag in XML, make sure we are compatible here. - ETag: "\"" + obj.ETag + "\"", - } - return c -} - -// GenerateListObjectsV2Response Generates an ListObjectsV2 response for the said bucket with other enumerated options. -//func GenerateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, isTruncated bool, maxKeys int, objects []object.Object, prefixes []string) ListObjectsV2Response { -// contents := make([]Object, 0, len(objects)) -// id := consts.DefaultOwnerID -// name := consts.DisplayName -// owner := s3.Owner{ -// ID: &id, -// DisplayName: &name, -// } -// data := ListObjectsV2Response{} -// -// for _, object := range objects { -// content := Object{} -// if object.Name == "" { -// continue -// } -// content.Key = utils.S3EncodeName(object.Name, encodingType) -// content.LastModified = object.ModTime.UTC().Format(consts.Iso8601TimeFormat) -// if object.ETag != "" { -// content.ETag = "\"" + object.ETag + "\"" -// } -// content.Size = object.Size -// content.Owner = owner -// content.CID = object.CID -// contents = append(contents, content) -// } -// data.Name = bucket -// data.Contents = contents -// -// data.EncodingType = encodingType -// data.StartAfter = utils.S3EncodeName(startAfter, encodingType) -// data.Delimiter = utils.S3EncodeName(delimiter, encodingType) -// data.Prefix = utils.S3EncodeName(prefix, encodingType) -// data.MaxKeys = maxKeys -// data.ContinuationToken = base64.StdEncoding.EncodeToString([]byte(token)) -// data.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(nextToken)) -// data.IsTruncated = isTruncated -// -// commonPrefixes := make([]CommonPrefix, 0, len(prefixes)) -// for _, prefix := range prefixes { -// prefixItem := CommonPrefix{} -// prefixItem.Prefix = utils.S3EncodeName(prefix, encodingType) -// commonPrefixes = append(commonPrefixes, prefixItem) -// } -// data.CommonPrefixes = commonPrefixes -// data.KeyCount = len(data.Contents) + len(data.CommonPrefixes) -// return data -//} - -// generates an ListObjectsV1 response for the said bucket with other enumerated options. -//func GenerateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp object.ObjectsList) ListObjectsResponse { -// contents := make([]Object, 0, len(resp.Objects)) -// id := consts.DefaultOwnerID -// name := consts.DisplayName -// owner := s3.Owner{ -// ID: &id, -// DisplayName: &name, -// } -// data := ListObjectsResponse{} -// -// for _, object := range resp.Objects { -// content := Object{} -// if object.Name == "" { -// continue -// } -// content.Key = utils.S3EncodeName(object.Name, encodingType) -// content.LastModified = object.ModTime.UTC().Format(consts.Iso8601TimeFormat) -// if object.ETag != "" { -// content.ETag = "\"" + object.ETag + "\"" -// } -// content.CID = object.CID -// content.Size = object.Size -// content.StorageClass = "" -// content.Owner = owner -// contents = append(contents, content) -// } -// data.Name = bucket -// data.Contents = contents -// -// data.EncodingType = encodingType -// data.Prefix = utils.S3EncodeName(prefix, encodingType) -// data.Marker = utils.S3EncodeName(marker, encodingType) -// data.Delimiter = utils.S3EncodeName(delimiter, encodingType) -// data.MaxKeys = maxKeys -// data.NextMarker = utils.S3EncodeName(resp.NextMarker, encodingType) -// data.IsTruncated = resp.IsTruncated -// -// prefixes := make([]CommonPrefix, 0, len(resp.Prefixes)) -// for _, prefix := range resp.Prefixes { -// prefixItem := CommonPrefix{} -// prefixItem.Prefix = utils.S3EncodeName(prefix, encodingType) -// prefixes = append(prefixes, prefixItem) -// } -// data.CommonPrefixes = prefixes -// return data -//} -// diff --git a/s3/responses/types_common.go b/s3/responses/types_common.go deleted file mode 100644 index 66522789b..000000000 --- a/s3/responses/types_common.go +++ /dev/null @@ -1 +0,0 @@ -package responses diff --git a/s3/routers/routers_options.go b/s3/routers/options.go similarity index 100% rename from s3/routers/routers_options.go rename to s3/routers/options.go diff --git a/s3/s3.go b/s3/s3.go index a37ebd7a5..6ab69c45e 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -2,7 +2,7 @@ package s3 import ( config "github.com/bittorrent/go-btfs-config" - "github.com/bittorrent/go-btfs/chain" + "github.com/bittorrent/go-btfs/s3/ctxmu" "github.com/bittorrent/go-btfs/s3/handlers" "github.com/bittorrent/go-btfs/s3/providers" "github.com/bittorrent/go-btfs/s3/routers" @@ -10,6 +10,7 @@ import ( "github.com/bittorrent/go-btfs/s3/services/accesskey" "github.com/bittorrent/go-btfs/s3/services/object" "github.com/bittorrent/go-btfs/s3/services/sign" + "github.com/bittorrent/go-btfs/transaction/storage" "sync" ) @@ -18,31 +19,31 @@ var ( once sync.Once ) -func initProviders() { +func InitProviders(stateStore storage.StateStorer) { once.Do(func() { - sstore := providers.NewStorageStateStoreProxy(chain.StateStore) + sstore := providers.NewStorageStateStoreProxy(stateStore) fstore := providers.NewBtfsAPI("") ps = providers.NewProviders(sstore, fstore) }) } func GetProviders() *providers.Providers { - initProviders() return ps } func NewServer(cfg config.S3CompatibleAPI) *server.Server { - // providers - initProviders() + // lock global multiple keys read write lock + lock := ctxmu.NewDefaultMultiCtxRWMutex() // services - acksvc := accesskey.NewService(ps) sigsvc := sign.NewService() - objsvc := object.NewService(ps) + acksvc := accesskey.NewService(ps, accesskey.WithLock(lock)) + objsvc := object.NewService(ps, object.WithLock(lock)) // handlers hs := handlers.NewHandlers( - acksvc, sigsvc, objsvc, handlers.WithHeaders(cfg.HTTPHeaders), + acksvc, sigsvc, objsvc, + handlers.WithHeaders(cfg.HTTPHeaders), ) // routers diff --git a/s3/server/server_options.go b/s3/server/options.go similarity index 100% rename from s3/server/server_options.go rename to s3/server/options.go diff --git a/s3/services/accesskey/service_instance.go b/s3/services/accesskey/instance.go similarity index 100% rename from s3/services/accesskey/service_instance.go rename to s3/services/accesskey/instance.go diff --git a/s3/services/accesskey/options.go b/s3/services/accesskey/options.go new file mode 100644 index 000000000..f959d6560 --- /dev/null +++ b/s3/services/accesskey/options.go @@ -0,0 +1,40 @@ +package accesskey + +import ( + "github.com/bittorrent/go-btfs/s3/ctxmu" + "time" +) + +const ( + defaultSecretLength = 32 + defaultStoreKeyPrefix = "access-keys:" + defaultWaitLockTimout = 2 * time.Minute +) + +var defaultLock = ctxmu.NewDefaultMultiCtxRWMutex() + +type Option func(svc *service) + +func WithSecretLength(length int) Option { + return func(svc *service) { + svc.secretLength = length + } +} + +func WithStoreKeyPrefix(prefix string) Option { + return func(svc *service) { + svc.storeKeyPrefix = prefix + } +} + +func WithWaitLockTimout(timout time.Duration) Option { + return func(svc *service) { + svc.waitLockTimeout = timout + } +} + +func WithLock(lock ctxmu.MultiCtxRWLocker) Option { + return func(svc *service) { + svc.lock = lock + } +} diff --git a/s3/services/accesskey/service.go b/s3/services/accesskey/service.go index 6ba1f91e2..5a7f5c3ee 100644 --- a/s3/services/accesskey/service.go +++ b/s3/services/accesskey/service.go @@ -11,29 +11,24 @@ import ( "time" ) -const ( - defaultSecretLength = 32 - defaultStoreKeyPrefix = "access-keys:" - defaultUpdateTimeoutMS = 200 -) - var _ Service = (*service)(nil) type service struct { - providers providers.Providerser - secretLength int - storeKeyPrefix string - locks *ctxmu.MultiCtxRWMutex - updateTimeout time.Duration + providers providers.Providerser + secretLength int + storeKeyPrefix string + lock ctxmu.MultiCtxRWLocker + waitLockTimeout time.Duration } + func NewService(providers providers.Providerser, options ...Option) Service { svc := &service{ - providers: providers, - secretLength: defaultSecretLength, - storeKeyPrefix: defaultStoreKeyPrefix, - locks: ctxmu.NewDefaultMultiCtxRWMutex(), - updateTimeout: time.Duration(defaultUpdateTimeoutMS) * time.Millisecond, + providers: providers, + secretLength: defaultSecretLength, + storeKeyPrefix: defaultStoreKeyPrefix, + lock: defaultLock, + waitLockTimeout: defaultWaitLockTimout, } for _, option := range options { option(svc) @@ -137,14 +132,14 @@ type updateArgs struct { } func (svc *service) update(key string, args *updateArgs) (err error) { - ctx, cancel := context.WithTimeout(context.Background(), svc.updateTimeout) + ctx, cancel := context.WithTimeout(context.Background(), svc.waitLockTimeout) defer cancel() - err = svc.locks.Lock(ctx, key) + err = svc.lock.Lock(ctx, key) if err != nil { return } - defer svc.locks.Unlock(key) + defer svc.lock.Unlock(key) record := &AccessKey{} stk := svc.getStoreKey(key) diff --git a/s3/services/accesskey/service_options.go b/s3/services/accesskey/service_options.go deleted file mode 100644 index 25f1617a6..000000000 --- a/s3/services/accesskey/service_options.go +++ /dev/null @@ -1,15 +0,0 @@ -package accesskey - -type Option func(svc *service) - -func WithSecretLength(length int) Option { - return func(svc *service) { - svc.secretLength = length - } -} - -func WithStoreKeyPrefix(prefix string) Option { - return func(svc *service) { - svc.storeKeyPrefix = prefix - } -} diff --git a/s3/services/object/options.go b/s3/services/object/options.go index 05c4dd9a3..7e6b80f58 100644 --- a/s3/services/object/options.go +++ b/s3/services/object/options.go @@ -7,9 +7,9 @@ import ( const ( defaultKeySeparator = "/" - defaultBucketSpace = "bkt" - defaultObjectSpace = "obj" - defaultUploadSpace = "upl" + defaultBucketSpace = "s3:bkt" + defaultObjectSpace = "s3:obj" + defaultUploadSpace = "s3:upl" defaultOperationTimeout = 5 * time.Minute defaultCloseBodyTimeout = 10 * time.Minute ) diff --git a/s3/services/sign/service_options.go b/s3/services/sign/options.go similarity index 100% rename from s3/services/sign/service_options.go rename to s3/services/sign/options.go diff --git a/settlement/swap/vault/cashout.go b/settlement/swap/vault/cashout.go index c6ee34bb5..89540735d 100644 --- a/settlement/swap/vault/cashout.go +++ b/settlement/swap/vault/cashout.go @@ -131,7 +131,7 @@ func cashoutActionKey(vault common.Address, token common.Address) string { // // output, err := s.transactionService.Call(ctx, &transaction.TxRequest{ // To: &vault, -// Data: callData, +// data: callData, // }) // if err != nil { // return nil, err @@ -191,7 +191,7 @@ func (s *cashoutService) CashCheque(ctx context.Context, vault, recipient common //} //request := &transaction.TxRequest{ // To: &vault, - // Data: callData, + // data: callData, // Value: big.NewInt(0), // Description: "cheque cashout", //} diff --git a/settlement/swap/vault/factory_test.go b/settlement/swap/vault/factory_test.go index 652eb87e3..f52c352ec 100644 --- a/settlement/swap/vault/factory_test.go +++ b/settlement/swap/vault/factory_test.go @@ -207,12 +207,12 @@ func TestFactoryVerifyVault(t *testing.T) { // Status: 1, // Logs: []*types.Log{ // { -// Data: logData, +// data: logData, // }, // { // Address: factoryAddress, // Topics: []common.Hash{simpleSwapDeployedEvent.ID}, -// Data: logData, +// data: logData, // }, // }, // }, nil diff --git a/settlement/swap/vault/vault.go b/settlement/swap/vault/vault.go index d8fe95800..d4733a3d2 100644 --- a/settlement/swap/vault/vault.go +++ b/settlement/swap/vault/vault.go @@ -526,7 +526,7 @@ func (s *service) LastCheques(token common.Address) (map[common.Address]*SignedC // // request := &transaction.TxRequest{ // To: &s.address, -// Data: callData, +// data: callData, // Value: big.NewInt(0), // Description: fmt.Sprintf("vault withdrawal of %d WBTT", amount), // }