From 2e4bf8d60508d1191c0c90b77d213aea04002ed1 Mon Sep 17 00:00:00 2001 From: Jackson Tian Date: Fri, 2 Jul 2021 12:17:40 +0800 Subject: [PATCH 1/5] Create go.yml --- .github/workflows/go.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 000000000..9b4e15675 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,25 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Build + run: make build + + - name: Test + run: make test From 711d9de0c51cca21aac1205d5862872906f1e8a7 Mon Sep 17 00:00:00 2001 From: Jackson Tian Date: Fri, 2 Jul 2021 14:15:27 +0800 Subject: [PATCH 2/5] fix test cases --- config/legacy.go | 12 ++++++++---- config/legacy_test.go | 8 +++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/config/legacy.go b/config/legacy.go index 66ef32462..2458a756f 100644 --- a/config/legacy.go +++ b/config/legacy.go @@ -27,13 +27,17 @@ func MigrateLegacyConfiguration() (conf *Configuration, err error) { return } - conf, err = MigrateCredentials(path) - if err != nil { + conf, migrateErr := MigrateCredentials(path) + if migrateErr != nil { return } path = hookGetHomePath(GetHomePath)() + "/.aliyuncli/configure" - err = MigrateConfigure(path, conf) + mfErr := MigrateConfigure(path, conf) + if mfErr != nil { + return + } + return } @@ -97,5 +101,5 @@ func MigrateConfigure(path string, conf *Configuration) (err error) { } } - return + return nil } diff --git a/config/legacy_test.go b/config/legacy_test.go index 2987e0a12..268ad7ef1 100644 --- a/config/legacy_test.go +++ b/config/legacy_test.go @@ -62,9 +62,9 @@ func TestMigrateConfigure(t *testing.T) { conf := &Configuration{CurrentProfile: "default", Profiles: []Profile{Profile{Name: "default", Mode: AK, AccessKeyId: "default_aliyun_access_key_id", AccessKeySecret: "default_aliyun_access_key_secret", OutputFormat: "json"}, Profile{Name: "aaa", Mode: AK, AccessKeyId: "sdf", AccessKeySecret: "ddf", OutputFormat: "json"}}} err := MigrateConfigure("http://nici", conf) if runtime.GOOS == "windows" { - assert.Equal(t, "parse failed: open http://nici: The filename, directory name, or volume label syntax is incorrect.", err.Error()) + assert.Equal(t, "parse failed: open http://nici: The filename, directory name, or volume label syntax is incorrect.\n", err.Error()) } else { - assert.Equal(t, "parse failed: open http://nici: no such file or directory", err.Error()) + assert.Equal(t, "parse failed: open http://nici: no such file or directory\n", err.Error()) } test, err := os.Create("testconf.ini") @@ -101,11 +101,13 @@ func TestMigrateLegacyConfiguration(t *testing.T) { os.RemoveAll("./.aliyuncli") hookGetHomePath = orighookGetHomePath }() + hookGetHomePath = func(fn func() string) func() string { return func() string { return "." } } + err := os.Mkdir("./.aliyuncli", os.ModePerm) assert.Nil(t, err) @@ -126,5 +128,5 @@ func TestMigrateLegacyConfiguration(t *testing.T) { test.Close() conf, err := MigrateLegacyConfiguration() assert.Nil(t, err) - assert.Nil(t, conf) + assert.NotNil(t, conf) } From 57b079297b7355ecbe50788c01f83fe3754131e0 Mon Sep 17 00:00:00 2001 From: Jackson Tian Date: Fri, 2 Jul 2021 14:29:43 +0800 Subject: [PATCH 3/5] improve github action --- .github/workflows/go.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9b4e15675..064d9b378 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -12,6 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # fetch submodule for aliyun-openapi-meta + - run: git submodule update --init --recursive - name: Set up Go uses: actions/setup-go@v2 @@ -22,4 +24,6 @@ jobs: run: make build - name: Test - run: make test + run: go vet ./cli/... ./config/... ./i18n/... ./main/... ./openapi/... ./oss/... ./resource/... + - run: go test -race -coverprofile=coverage.txt -covermode=atomic ./cli/... ./config/... ./i18n/... ./meta/... ./main/... ./openapi/... ./resource/... + - run: test -z $ACCESS_KEY_ID -a -z $ACCESS_KEY_SECRET || bash ./integration/ecs_test From 83f74bc03d8c25bd4e235d8c4570f2a5f78eabcc Mon Sep 17 00:00:00 2001 From: Jackson Tian Date: Fri, 2 Jul 2021 11:58:17 +0800 Subject: [PATCH 4/5] upgrade ossutil to v1.7.4 --- oss/lib/allpart_size.go | 8 + oss/lib/append_file.go | 8 + oss/lib/bucket_cors.go | 8 + oss/lib/bucket_encryption.go | 8 + oss/lib/bucket_inventory.go | 8 + oss/lib/bucket_lifecycle.go | 8 + oss/lib/bucket_logging.go | 8 + oss/lib/bucket_policy.go | 8 + oss/lib/bucket_qos.go | 8 + oss/lib/bucket_referer.go | 8 + oss/lib/bucket_tagging.go | 8 + oss/lib/bucket_versioning.go | 8 + oss/lib/bucket_website.go | 8 + oss/lib/bucket_worm.go | 8 + oss/lib/cat.go | 8 + oss/lib/command.go | 145 +++++- oss/lib/command_manager.go | 1 + oss/lib/command_test.go | 19 + oss/lib/config.go | 1 + oss/lib/config_helper.go | 14 + oss/lib/const.go | 10 +- oss/lib/cors_options.go | 8 + oss/lib/cp.go | 11 + oss/lib/cp_test.go | 808 ++++++++++++++++++++++++++++++++ oss/lib/create_symlink.go | 8 + oss/lib/du.go | 8 + oss/lib/hash_test.go | 2 +- oss/lib/listpart.go | 8 + oss/lib/ls.go | 8 + oss/lib/ls_test.go | 2 +- oss/lib/mb.go | 8 + oss/lib/mkdir.go | 8 + oss/lib/object_tagging.go | 8 + oss/lib/object_tagging_test.go | 14 +- oss/lib/option.go | 24 + oss/lib/probe.go | 8 + oss/lib/read_symlink.go | 8 + oss/lib/request_payment.go | 8 + oss/lib/request_payment_test.go | 3 + oss/lib/restore.go | 8 + oss/lib/revert_versioning.go | 8 + oss/lib/rm.go | 8 + oss/lib/set_acl.go | 8 + oss/lib/set_acl_test.go | 14 +- oss/lib/set_meta.go | 8 + oss/lib/signurl.go | 8 + oss/lib/stat.go | 8 + oss/lib/stat_test.go | 1 + oss/lib/sync.go | 8 + oss/lib/user_qos.go | 8 + oss/lib/util_sts.go | 183 ++++++++ oss/lib/util_sts_test.go | 128 +++++ 52 files changed, 1631 insertions(+), 29 deletions(-) create mode 100644 oss/lib/util_sts.go create mode 100644 oss/lib/util_sts_test.go diff --git a/oss/lib/allpart_size.go b/oss/lib/allpart_size.go index 426f33e80..0ed941aec 100644 --- a/oss/lib/allpart_size.go +++ b/oss/lib/allpart_size.go @@ -95,6 +95,14 @@ var allPartSizeCommand = AllPartSizeCommand{ OptionEncodingType, OptionLogLevel, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/append_file.go b/oss/lib/append_file.go index d7d45260f..d4fc185a2 100644 --- a/oss/lib/append_file.go +++ b/oss/lib/append_file.go @@ -150,6 +150,14 @@ var appendFileCommand = AppendFileCommand{ OptionLogLevel, OptionRequestPayer, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_cors.go b/oss/lib/bucket_cors.go index becd4de02..575497e2a 100644 --- a/oss/lib/bucket_cors.go +++ b/oss/lib/bucket_cors.go @@ -146,6 +146,14 @@ var corsCommand = CorsCommand{ OptionMethod, OptionLogLevel, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_encryption.go b/oss/lib/bucket_encryption.go index c324b26a8..f5fe9d307 100644 --- a/oss/lib/bucket_encryption.go +++ b/oss/lib/bucket_encryption.go @@ -135,6 +135,14 @@ var bucketEncryptionCommand = BucketEncryptionCommand{ OptionKMSMasterKeyID, OptionKMSDataEncryption, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_inventory.go b/oss/lib/bucket_inventory.go index 69d20eb6b..3b5de3086 100644 --- a/oss/lib/bucket_inventory.go +++ b/oss/lib/bucket_inventory.go @@ -207,6 +207,14 @@ var bucketInventoryCommand = BucketInventoryCommand{ OptionMethod, OptionMarker, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_lifecycle.go b/oss/lib/bucket_lifecycle.go index 219824395..6a6391417 100644 --- a/oss/lib/bucket_lifecycle.go +++ b/oss/lib/bucket_lifecycle.go @@ -165,6 +165,14 @@ var bucketLifeCycleCommand = BucketLifeCycleCommand{ OptionLogLevel, OptionMethod, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_logging.go b/oss/lib/bucket_logging.go index 0a8494bc6..a7c9f32eb 100644 --- a/oss/lib/bucket_logging.go +++ b/oss/lib/bucket_logging.go @@ -125,6 +125,14 @@ var bucketLogCommand = BucketLogCommand{ OptionMethod, OptionLogLevel, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_policy.go b/oss/lib/bucket_policy.go index 50bbbbd61..97cf4dd5f 100644 --- a/oss/lib/bucket_policy.go +++ b/oss/lib/bucket_policy.go @@ -162,6 +162,14 @@ var bucketPolicyCommand = BucketPolicyCommand{ OptionLogLevel, OptionMethod, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_qos.go b/oss/lib/bucket_qos.go index c18eda380..ad29b43fa 100644 --- a/oss/lib/bucket_qos.go +++ b/oss/lib/bucket_qos.go @@ -151,6 +151,14 @@ var bucketQosCommand = BucketQosCommand{ OptionLogLevel, OptionMethod, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_referer.go b/oss/lib/bucket_referer.go index f35f9610b..efc012aee 100644 --- a/oss/lib/bucket_referer.go +++ b/oss/lib/bucket_referer.go @@ -131,6 +131,14 @@ var bucketRefererCommand = BucketRefererCommand{ OptionDisableEmptyReferer, OptionMethod, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_tagging.go b/oss/lib/bucket_tagging.go index 9817b4452..96246aa22 100644 --- a/oss/lib/bucket_tagging.go +++ b/oss/lib/bucket_tagging.go @@ -115,6 +115,14 @@ var bucketTagCommand = BucketTagCommand{ OptionMethod, OptionLogLevel, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_versioning.go b/oss/lib/bucket_versioning.go index 9ec6684f7..977f0bb4a 100644 --- a/oss/lib/bucket_versioning.go +++ b/oss/lib/bucket_versioning.go @@ -107,6 +107,14 @@ var bucketVersioningCommand = BucketVersioningCommand{ OptionMethod, OptionLogLevel, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_website.go b/oss/lib/bucket_website.go index 19131f9d5..088c3823c 100644 --- a/oss/lib/bucket_website.go +++ b/oss/lib/bucket_website.go @@ -238,6 +238,14 @@ var bucketWebsiteCommand = BucketWebSiteCommand{ OptionLogLevel, OptionMethod, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/bucket_worm.go b/oss/lib/bucket_worm.go index 252e741ea..77aa75657 100644 --- a/oss/lib/bucket_worm.go +++ b/oss/lib/bucket_worm.go @@ -118,6 +118,14 @@ var wormCommand = WormCommand{ OptionProxyPwd, OptionLogLevel, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/cat.go b/oss/lib/cat.go index bf44bd9d6..9875e6c40 100644 --- a/oss/lib/cat.go +++ b/oss/lib/cat.go @@ -103,6 +103,14 @@ var catCommand = CatCommand{ OptionVersionId, OptionRequestPayer, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/command.go b/oss/lib/command.go index 5b72515cd..0961bdaf6 100644 --- a/oss/lib/command.go +++ b/oss/lib/command.go @@ -289,10 +289,25 @@ func (cmd *Command) ossClient(bucket string) (*oss.Client, error) { proxyHost, _ := GetString(OptionProxyHost, cmd.options) proxyUser, _ := GetString(OptionProxyUser, cmd.options) proxyPwd, _ := GetString(OptionProxyPwd, cmd.options) - ecsUrl, _ := cmd.getEcsRamAkService() + + mode, _ := GetString(OptionMode, cmd.options) + ecsRoleName, _ := GetString(OptionECSRoleName, cmd.options) + + strTokenTimeout, _ := GetString(OptionTokenTimeout, cmd.options) + ramRoleArn, _ := GetString(OptionRamRoleArn, cmd.options) + roleSessionName, _ := GetString(OptionRoleSessionName, cmd.options) + + strReadTimeout, _ := GetString(OptionReadTimeout, cmd.options) + strConnectTimeout, _ := GetString(OptionConnectTimeout, cmd.options) + + stsRegion, _ := GetString(OptionSTSRegion, cmd.options) + + ecsUrl := "" + localHost, _ := GetString(OptionLocalHost, cmd.options) bPassword, _ := GetBool(OptionPassword, cmd.options) + if bPassword { if cmd.inputKeySecret == "" { strPwd, err := GetPassword("input access key secret:") @@ -305,17 +320,116 @@ func (cmd *Command) ossClient(bucket string) (*oss.Client, error) { accessKeySecret = cmd.inputKeySecret } - if accessKeyID == "" && ecsUrl == "" { - return nil, fmt.Errorf("accessKeyID and ecsUrl are both empty") - } + options := []oss.ClientOption{} + + if strings.EqualFold(mode, "AK") { + if err := cmd.checkCredentials(endpoint, accessKeyID, accessKeySecret); err != nil { + return nil, err + } + } else if strings.EqualFold(mode, "StsToken") { - if ecsUrl == "" { if err := cmd.checkCredentials(endpoint, accessKeyID, accessKeySecret); err != nil { return nil, err } + if stsToken == "" { + return nil, fmt.Errorf("stsToken is empty") + } + options = append(options, oss.SecurityToken(stsToken)) + + } else if strings.EqualFold(mode, "RamRoleArn") { + if err := cmd.checkCredentials(endpoint, accessKeyID, accessKeySecret); err != nil { + return nil, err + } + if ramRoleArn == "" { + ramRoleArn, _ = cmd.getRamRoleArn() + } + if ramRoleArn == "" { + return nil, fmt.Errorf("ramRoleArn is empty") + } + if roleSessionName == "" { + roleSessionName = "SessNameRand" + randStr(5) + } + // sts.NewClient(stsaccessID, stsaccessKey, stsARN, "oss_test_sess") + stsClient := NewClient(accessKeyID, accessKeySecret, ramRoleArn, roleSessionName) + + if strTokenTimeout == "" { + strTokenTimeout = "3600" + } + intTokenTimeout, err := strconv.Atoi(strTokenTimeout) + if err != nil { + return nil, err + } + TokenTimeout := uint(intTokenTimeout) + + stsEndPoint := "" + if stsRegion == "" { + stsEndPoint = "" + } else { + stsEndPoint = "https://sts." + stsRegion + ".aliyuncs.com" + } + + resp, err := stsClient.AssumeRole(TokenTimeout, stsEndPoint) + if err != nil { + return nil, err + } + + accessKeyID = resp.Credentials.AccessKeyId + accessKeySecret = resp.Credentials.AccessKeySecret + stsToken = resp.Credentials.SecurityToken + options = append(options, oss.SecurityToken(stsToken)) + + } else if strings.EqualFold(mode, "EcsRamRole") { + if ecsRoleName != "" { + ecsUrl = "http://100.100.100.200/latest/meta-data/Ram/security-credentials/" + ecsRoleName + } else { + ecsUrl, _ = cmd.getEcsRamAkService() + } + + if ecsUrl == "" { + return nil, fmt.Errorf("ecsUrl is empty") + } + ecsRoleAKBuild := EcsRoleAKBuild{url: ecsUrl} + options = append(options, oss.SetCredentialsProvider(&ecsRoleAKBuild)) + + } else if mode == "" { + + ecsUrl, _ = cmd.getEcsRamAkService() + if accessKeyID == "" && ecsUrl == "" { + return nil, fmt.Errorf("accessKeyID and ecsUrl are both empty") + } + if ecsUrl == "" { + if err := cmd.checkCredentials(endpoint, accessKeyID, accessKeySecret); err != nil { + return nil, err + } + } + if accessKeyID == "" { + LogInfo("using user ak service:%s\n", ecsUrl) + ecsRoleAKBuild := EcsRoleAKBuild{url: ecsUrl} + options = append(options, oss.SetCredentialsProvider(&ecsRoleAKBuild)) + } + + if stsToken != "" { + options = append(options, oss.SecurityToken(stsToken)) + } } - options := []oss.ClientOption{oss.UseCname(isCname), oss.SecurityToken(stsToken), oss.UserAgent(getUserAgent()), oss.Timeout(120, 1200)} + if strConnectTimeout == "" { + strConnectTimeout = "120" + } + if strReadTimeout == "" { + strReadTimeout = "1200" + } + connectTimeout, err := strconv.ParseInt(strConnectTimeout, 10, 64) + if err != nil { + return nil, err + } + readTimeout, err := strconv.ParseInt(strReadTimeout, 10, 64) + if err != nil { + return nil, err + } + + options = append(options, oss.UseCname(isCname), oss.UserAgent(getUserAgent()), oss.Timeout(connectTimeout, readTimeout)) + if disableCRC64 { options = append(options, oss.EnableCRC(false)) } else { @@ -344,12 +458,6 @@ func (cmd *Command) ossClient(bucket string) (*oss.Client, error) { options = append(options, oss.SetLogger(utilLogger)) } - if accessKeyID == "" { - LogInfo("using user ak service:%s\n", ecsUrl) - ecsRoleAKBuild := EcsRoleAKBuild{url: ecsUrl} - options = append(options, oss.SetCredentialsProvider(&ecsRoleAKBuild)) - } - client, err := oss.New(endpoint, accessKeyID, accessKeySecret, options...) if err != nil { return nil, err @@ -398,6 +506,19 @@ func (cmd *Command) getEcsRamAkService() (string, bool) { return "", false } +func (cmd *Command) getRamRoleArn() (string, bool) { + if arnMap, ok := cmd.configOptions[CREDSection]; ok { + if strArn, ok := arnMap.(map[string]string)[ItemRamRoleArn]; ok { + if strArn != "" { + return strArn, true + } else { + return "", false + } + } + } + return "", false +} + func (cmd *Command) getEndpoint(bucket string) (string, bool) { if cnameMap, ok := cmd.configOptions[BucketCnameSection]; ok { if endpoint, ok := cnameMap.(map[string]string)[bucket]; ok { diff --git a/oss/lib/command_manager.go b/oss/lib/command_manager.go index c5e819d4b..c09117cf6 100644 --- a/oss/lib/command_manager.go +++ b/oss/lib/command_manager.go @@ -113,6 +113,7 @@ func (cm *CommandManager) Init() { // RunCommand select command from command map, initialize command and run command func (cm *CommandManager) RunCommand(commandName string, args []string, options OptionMapType) (bool, error) { + if cmd, ok := cm.commandMap[commandName]; ok { if err := cmd.(Commander).Init(args, options); err != nil { return false, err diff --git a/oss/lib/command_test.go b/oss/lib/command_test.go index 3f62c70d2..b2336537d 100644 --- a/oss/lib/command_test.go +++ b/oss/lib/command_test.go @@ -39,7 +39,11 @@ var ( proxyUser = "" proxyPwd = "" accountID = "" + stsAccessID = "" + stsAccessKeySecret = "" stsARN = "" + ecsRoleName = "" + stsRegion = "" ) var ( @@ -136,9 +140,24 @@ func SetUpCredential() { if accountID == "" { accountID = os.Getenv("OSS_TEST_ACCOUNT_ID") } + if stsToken == "" { + stsToken = os.Getenv("OSS_TEST_STS_TOKEN") + } + if stsAccessID == "" { + stsAccessID = os.Getenv("OSS_TEST_STS_ID") + } + if stsAccessKeySecret == "" { + stsAccessKeySecret = os.Getenv("OSS_TEST_STS_KEY") + } if stsARN == "" { stsARN = os.Getenv("OSS_TEST_STS_ARN") } + if ecsRoleName == "" { + ecsRoleName = os.Getenv("OSS_TEST_ECS_ROLE_NAME") + } + if stsRegion == "" { + stsRegion = os.Getenv("OSS_TEST_STS_REGION") + } } func (s *OssutilCommandSuite) SetUpBucketEnv(c *C) { diff --git a/oss/lib/config.go b/oss/lib/config.go index 6bbd36c6c..a8e3e2b97 100644 --- a/oss/lib/config.go +++ b/oss/lib/config.go @@ -307,6 +307,7 @@ var configCommand = ConfigCommand{ OptionOutputDir, OptionLanguage, OptionLogLevel, + OptionRamRoleArn, }, }, } diff --git a/oss/lib/config_helper.go b/oss/lib/config_helper.go index 59809a4d2..ab1852386 100644 --- a/oss/lib/config_helper.go +++ b/oss/lib/config_helper.go @@ -25,6 +25,11 @@ const ( ItemEcsAk string = "ecsAk" ) +// config items in section Credentials +const ( + ItemRamRoleArn string = "ramRoleArn" +) + type configOption struct { showNames []string cfInteractive bool @@ -41,6 +46,7 @@ var CredOptionList = []string{ OptionAccessKeySecret, OptionSTSToken, OptionOutputDir, + OptionRamRoleArn, } // CredOptionMap allows alias name for options in Credentials section @@ -52,6 +58,7 @@ var CredOptionMap = map[string]configOption{ OptionAccessKeySecret: configOption{[]string{"accessKeySecret", "AccessKeySecret", "access_key_secret", "access_key", "accesskey", "access-key-secret", "access-key"}, true, false, "", ""}, OptionSTSToken: configOption{[]string{"stsToken", "ststoken", "STSToken", "sts_token", "sts-token"}, true, false, "", ""}, OptionOutputDir: configOption{[]string{"outputDir", "output-dir", "output_dir", "output_directory"}, false, true, "ossutil生成的文件的输出目录, ", "the directory to store files generated by ossutil, "}, + //OptionRamRoleArn: configOption{[]string{"ramRoleArn", "RamRoleArn", "RamRolearn", "ram-role-arn", "ram_role_arn", "role_arn", "rolearn", "arn"}, false, true, "", ""}, } // DecideConfigFile return the config file, if user not specified, return default one @@ -100,9 +107,15 @@ func readConfigFromFile(configFile string) (OptionMapType, error) { } credOptions := credSection.Options() + + //added + //configMap[CREDSection] = map[string]string{} + for name, option := range credOptions { if opName, ok := getOptionNameByStr(strings.TrimSpace(name)); ok { configMap[strings.TrimSpace(opName)] = strings.TrimSpace(option) + } else { + configMap[strings.TrimSpace(name)] = strings.TrimSpace(option) } } @@ -126,6 +139,7 @@ func readConfigFromFile(configFile string) (OptionMapType, error) { (configMap[sec]).(map[string]string)[strings.TrimSpace(ecsUrl)] = strings.TrimSpace(strUrl) } } + return configMap, nil } diff --git a/oss/lib/const.go b/oss/lib/const.go index 9e8c18d64..3dd2c5926 100644 --- a/oss/lib/const.go +++ b/oss/lib/const.go @@ -87,6 +87,14 @@ const ( OptionBackupDir = "backupDir" OptionPassword = "password" OptionBlockSize = "blockSize" + OptionMode = "mode" + OptionECSRoleName = "ecsRoleName" + OptionTokenTimeout = "tokenTimeout" + OptionRamRoleArn = "ramRoleArn" + OptionRoleSessionName = "roleSessionName" + OptionReadTimeout = "readTimeOut" + OptionConnectTimeout = "connectTimeOut" + OptionSTSRegion = "stsRegion" ) // the elements show in stat object @@ -132,7 +140,7 @@ const ( const ( Package string = "ossutil" ChannelBuf int = 1000 - Version string = "v1.7.3" + Version string = "v1.7.4" DefaultEndpoint string = "oss.aliyuncs.com" ChineseLanguage = "CH" EnglishLanguage = "EN" diff --git a/oss/lib/cors_options.go b/oss/lib/cors_options.go index 14fdf74b1..bc671a27a 100644 --- a/oss/lib/cors_options.go +++ b/oss/lib/cors_options.go @@ -91,6 +91,14 @@ var corsOptionsCommand = OptionsCommand{ OptionAcrMethod, OptionAcrHeaders, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/cp.go b/oss/lib/cp.go index fa4b6a0d3..7a8957619 100644 --- a/oss/lib/cp.go +++ b/oss/lib/cp.go @@ -1279,6 +1279,14 @@ var copyCommand = CopyCommand{ OptionDisableIgnoreError, OptionTagging, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } @@ -1423,6 +1431,9 @@ func (cc *CopyCommand) RunCommand() error { // create checkpoint dir if err := os.MkdirAll(cc.cpOption.cpDir, 0755); err != nil { + + // + //fmt.Printf("%s", cc.cpOption.cpDir) return err } diff --git a/oss/lib/cp_test.go b/oss/lib/cp_test.go index df71f3a9a..f9722fc50 100644 --- a/oss/lib/cp_test.go +++ b/oss/lib/cp_test.go @@ -5028,3 +5028,811 @@ func (s *OssutilCommandSuite) TestCPObjectWithInputPassword(c *C) { os.Remove(fileName) s.removeBucket(bucketName, true, c) } + +func (s *OssutilCommandSuite) TestCPObjectUnderAKmode(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + mode := "AK" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + } + + c.Log(options) + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + os.Remove(downFileName) + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderEcsmodeWithConfigEcsUrl(c *C) { + oldConfigStr, err := ioutil.ReadFile(configFile) + c.Assert(err, IsNil) + fd, _ := os.OpenFile(configFile, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + ecsAk := "http://100.100.100.200/latest/meta-data/Ram/security-credentials/" + ecsRoleName + configStr := "[Credentials]" + "\n" + "language=EN" + "\n" + "endpoint=" + endpoint + "\n" + configStr = configStr + "[AkService]" + "\n" + "ecsAk=" + ecsAk + fd.WriteString(configStr) + fd.Close() + + // create bucket + mode := "EcsRamRole" + + str := "" + bucketName := bucketNamePrefix + randLowStr(12) + command := "mb" + args := []string{CloudURLToString(bucketName, "")} + options := OptionMapType{ + "endpoint": &str, + "mode": &mode, + "configFile": &configFile, + } + _, err = cm.RunCommand(command, args, options) + c.Assert(err, IsNil) + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options = OptionMapType{ + "endpoint": &str, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + } + + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + + err = ioutil.WriteFile(configFile, []byte(oldConfigStr), 0664) + c.Assert(err, IsNil) + + os.Remove(downFileName) + os.Remove(testFileName) + + // rm bucket + command = "rm" + args = []string{CloudURLToString(bucketName, "")} + ok := true + options = OptionMapType{ + "endpoint": &str, + "configFile": &configFile, + "mode": &mode, + "bucket": &ok, + "force": &ok, + "allType": &ok, + } + _, err = cm.RunCommand(command, args, options) + c.Assert(err, NotNil) + + s.createFile(configFile, string(oldConfigStr), c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderEcsmodeWithEmptyEcsUrl(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + mode := "EcsRamRole" + routines := strconv.Itoa(Routines) + + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + } + + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, NotNil) + + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderEcsmodeWithRolename(c *C) { + mode := "EcsRamRole" + str := "" + + // create bucket + bucketName := bucketNamePrefix + randLowStr(12) + command := "mb" + args := []string{CloudURLToString(bucketName, "")} + options := OptionMapType{ + "endpoint": &str, + "mode": &mode, + "configFile": &configFile, + "ecsRoleName": &ecsRoleName, + } + _, err := cm.RunCommand(command, args, options) + c.Assert(err, IsNil) + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options = OptionMapType{ + "endpoint": &str, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + "ecsRoleName": &ecsRoleName, + } + + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + os.Remove(downFileName) + os.Remove(testFileName) + + // rm bucket + command = "rm" + args = []string{CloudURLToString(bucketName, "")} + ok := true + options = OptionMapType{ + "endpoint": &str, + "configFile": &configFile, + "mode": &mode, + "ecsRoleName": &ecsRoleName, + "bucket": &ok, + "force": &ok, + "allType": &ok, + } + _, err = cm.RunCommand(command, args, options) + c.Assert(err, NotNil) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderRamRoleArnmodeWithArn(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + mode := "RamRoleArn" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &stsAccessID, + "accessKeySecret": &stsAccessKeySecret, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + "ramRoleArn": &stsARN, + } + + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + os.Remove(downFileName) + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderRamRoleArnmodeWithConfigArn(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + oldConfigStr, err := ioutil.ReadFile(configFile) + c.Assert(err, IsNil) + f, err := os.OpenFile(ConfigFile, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0660) + configStr := "ramRoleArn=" + stsARN + + f.WriteString(configStr) + f.Close() + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + mode := "RamRoleArn" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &stsAccessID, + "accessKeySecret": &stsAccessKeySecret, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + "ramRoleArn": &str, + } + + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + + err = ioutil.WriteFile(configFile, []byte(oldConfigStr), 0664) + c.Assert(err, IsNil) + + os.Remove(downFileName) + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) + +} + +func (s *OssutilCommandSuite) TestCPObjectUnderRamRoleArnmodeWithEmptyArn(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + mode := "RamRoleArn" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &stsAccessID, + "accessKeySecret": &stsAccessKeySecret, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + "ramRoleArn": &str, + } + + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, NotNil) + + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderRamRoleArnmodeWithTokenTimeout(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + mode := "RamRoleArn" + routines := strconv.Itoa(Routines) + tokenTimeout := "2000" + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &stsAccessID, + "accessKeySecret": &stsAccessKeySecret, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + "ramRoleArn": &stsARN, + "tokenTimeout": &tokenTimeout, + } + + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + os.Remove(downFileName) + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderRamRoleArnmodeWithStsRegion(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + str := "" + mode := "RamRoleArn" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &stsAccessID, + "accessKeySecret": &stsAccessKeySecret, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + "ramRoleArn": &stsARN, + "stsRegion": &stsRegion, + } + + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + os.Remove(downFileName) + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderNomode(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "routines": &routines, + "checkpointDir": &cpDir, + } + + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + os.Remove(downFileName) + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderNomodeWithEmptyAKIdAndEcsUrl(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + oldConfigStr, err := ioutil.ReadFile(configFile) + c.Assert(err, IsNil) + fd, _ := os.OpenFile(configFile, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + configStr := "[Credentials]" + "\n" + "language=EN" + "\n" + "endpoint=oss-cn-shenzhen.aliyuncs.com" + "\n" + + fd.WriteString(configStr) + fd.Close() + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "routines": &routines, + "checkpointDir": &cpDir, + } + + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, NotNil) + + err = ioutil.WriteFile(configFile, []byte(oldConfigStr), 0664) + c.Assert(err, IsNil) + + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderNomodeUsingEcsRoleAK(c *C) { + oldConfigStr, err := ioutil.ReadFile(configFile) + c.Assert(err, IsNil) + fd, _ := os.OpenFile(configFile, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + ecsAk := "http://100.100.100.200/latest/meta-data/Ram/security-credentials/" + ecsRoleName + configStr := "[Credentials]" + "\n" + "language=EN" + "\n" + "endpoint=" + endpoint + "\n" + configStr = configStr + "[AkService]" + "\n" + "ecsAk=" + ecsAk + fd.WriteString(configStr) + fd.Close() + + // create bucket + mode := "EcsRamRole" + str := "" + bucketName := bucketNamePrefix + randLowStr(12) + command := "mb" + args := []string{CloudURLToString(bucketName, "")} + options := OptionMapType{ + "endpoint": &str, + "mode": &mode, + "configFile": &configFile, + } + _, err = cm.RunCommand(command, args, options) + c.Assert(err, IsNil) + + // prepare file and object + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options = OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "routines": &routines, + "checkpointDir": &cpDir, + } + + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + os.Remove(downFileName) + os.Remove(testFileName) + + // rm bucket + command = "rm" + args = []string{CloudURLToString(bucketName, "")} + ok := true + options = OptionMapType{ + "endpoint": &str, + "configFile": &configFile, + "mode": &mode, + "bucket": &ok, + "force": &ok, + "allType": &ok, + } + _, err = cm.RunCommand(command, args, options) + c.Assert(err, NotNil) + s.removeBucket(bucketName, true, c) + s.createFile(configFile, string(oldConfigStr), c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderNomodeWithTimeOut(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + readTimeOut := "1000" + connectTimeOut := "100" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "routines": &routines, + "checkpointDir": &cpDir, + "readTimeOut": &readTimeOut, + "connectTimeOut": &connectTimeOut, + } + + c.Log(options) + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + os.Remove(downFileName) + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderSTStokenmode(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + client := NewClient(stsAccessID, stsAccessKeySecret, stsARN, "sts_test") + + resp, err := client.AssumeRole(900, "") + c.Assert(err, IsNil) + + temAccessID := resp.Credentials.AccessKeyId + temAccessKeySecret := resp.Credentials.AccessKeySecret + temSTSToken := resp.Credentials.SecurityToken + + c.Assert(temAccessID, Not(Equals), "") + c.Assert(temAccessKeySecret, Not(Equals), "") + c.Assert(temSTSToken, Not(Equals), "") + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + mode := "StsToken" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &temAccessID, + "accessKeySecret": &temAccessKeySecret, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + "stsToken": &temSTSToken, + } + + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // down object + downFileName := testFileName + ".down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + //check content + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(data, Equals, string(fileBody)) + os.Remove(downFileName) + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCPObjectUnderSTSTokenmodeWithEmptySTSToken(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + client := NewClient(stsAccessID, stsAccessKeySecret, stsARN, "sts_test") + + resp, err := client.AssumeRole(900, "") + c.Assert(err, IsNil) + + temAccessID := resp.Credentials.AccessKeyId + temAccessKeySecret := resp.Credentials.AccessKeySecret + temSTSToken := resp.Credentials.SecurityToken + + c.Assert(temAccessID, Not(Equals), "") + c.Assert(temAccessKeySecret, Not(Equals), "") + c.Assert(temSTSToken, Not(Equals), "") + + // prepare file and object + + // filename + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // begin cp file + object := randLowStr(12) + cpArgs := []string{testFileName, CloudURLToString(bucketName, object)} + + str := "" + mode := "StsToken" + routines := strconv.Itoa(Routines) + cpDir := CheckpointDir + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &temAccessID, + "accessKeySecret": &temAccessKeySecret, + "configFile": &configFile, + "mode": &mode, + "routines": &routines, + "checkpointDir": &cpDir, + "stsToken": &str, + } + + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, NotNil) + + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} diff --git a/oss/lib/create_symlink.go b/oss/lib/create_symlink.go index 58d5d58f8..b6fc744dc 100644 --- a/oss/lib/create_symlink.go +++ b/oss/lib/create_symlink.go @@ -118,6 +118,14 @@ var createSymlinkCommand = CreateSymlinkCommand{ OptionLogLevel, OptionRequestPayer, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/du.go b/oss/lib/du.go index 14d2b37c5..46dfa17c8 100644 --- a/oss/lib/du.go +++ b/oss/lib/du.go @@ -128,6 +128,14 @@ var duSizeCommand = DuCommand{ OptionAllversions, OptionPassword, OptionBlockSize, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/hash_test.go b/oss/lib/hash_test.go index 768ace065..807bc6115 100644 --- a/oss/lib/hash_test.go +++ b/oss/lib/hash_test.go @@ -1,8 +1,8 @@ package lib import ( - . "gopkg.in/check.v1" "os" + . "gopkg.in/check.v1" ) func (s *OssutilCommandSuite) TestErrorInputFile(c *C) { diff --git a/oss/lib/listpart.go b/oss/lib/listpart.go index ec20d3fe7..565a023be 100644 --- a/oss/lib/listpart.go +++ b/oss/lib/listpart.go @@ -95,6 +95,14 @@ var listPartCommand = ListPartCommand{ OptionEncodingType, OptionLogLevel, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/ls.go b/oss/lib/ls.go index ca2e879c8..b846fbcb6 100644 --- a/oss/lib/ls.go +++ b/oss/lib/ls.go @@ -401,6 +401,14 @@ var listCommand = ListCommand{ OptionAllversions, OptionVersionIdMarker, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/ls_test.go b/oss/lib/ls_test.go index fac6c8bf5..4cd0ce228 100644 --- a/oss/lib/ls_test.go +++ b/oss/lib/ls_test.go @@ -89,7 +89,7 @@ func (s *OssutilCommandSuite) TestListErrConfigFile(c *C) { func (s *OssutilCommandSuite) TestListConfigFile(c *C) { cfile := randStr(10) - data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\nretryTimes=%s", endpoint, accessKeyID, accessKeySecret, "errretry") + data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n", endpoint, accessKeyID, accessKeySecret) s.createFile(cfile, data, c) command := "ls" diff --git a/oss/lib/mb.go b/oss/lib/mb.go index 4633ca563..869799fb6 100644 --- a/oss/lib/mb.go +++ b/oss/lib/mb.go @@ -167,6 +167,14 @@ var makeBucketCommand = MakeBucketCommand{ OptionLogLevel, OptionRedundancyType, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/mkdir.go b/oss/lib/mkdir.go index ab20c01c9..6186bdd09 100644 --- a/oss/lib/mkdir.go +++ b/oss/lib/mkdir.go @@ -81,6 +81,14 @@ var mkdirCommand = MkdirCommand{ OptionLogLevel, OptionEncodingType, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/object_tagging.go b/oss/lib/object_tagging.go index a74e51247..4bedd5b39 100644 --- a/oss/lib/object_tagging.go +++ b/oss/lib/object_tagging.go @@ -145,6 +145,14 @@ var objectTagCommand = ObjectTagCommand{ OptionVersionId, OptionRequestPayer, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/object_tagging_test.go b/oss/lib/object_tagging_test.go index 3e3dc8557..ad720a06f 100644 --- a/oss/lib/object_tagging_test.go +++ b/oss/lib/object_tagging_test.go @@ -319,15 +319,15 @@ func (s *OssutilCommandSuite) TestObjectTaggingError(c *C) { strMethod = "put" tagArgs = []string{CloudURLToString(bucketName, object), tagInfo} _, err = cm.RunCommand("object-tagging", tagArgs, options) - c.Assert(err, NotNil) - - // tag value is error - tagInfo = "key1 value1" - delete(options, "payer") - strMethod = "put" + c.Assert(err, NotNil) + + // tag value is error + tagInfo = "key1 value1" + delete(options,"payer") + strMethod = "put" tagArgs = []string{CloudURLToString(bucketName, object), tagInfo} _, err = cm.RunCommand("object-tagging", tagArgs, options) - c.Assert(err, NotNil) + c.Assert(err, NotNil) os.Remove(fileName) s.removeBucket(bucketName, true, c) diff --git a/oss/lib/option.go b/oss/lib/option.go index 63b79a9b5..37d2e0f48 100644 --- a/oss/lib/option.go +++ b/oss/lib/option.go @@ -229,6 +229,30 @@ var OptionMap = map[string]Option{ OptionBlockSize: Option{"-B", "--block-size", "", OptionTypeString, "", "", "表示du命令字节显示的单位,取值可以为KB, MB, GB, TB", "specifies the unit of byte display for du command, the value can be KB, MB, GB, TB"}, + OptionMode: Option{"", "--mode", "", OptionTypeString, "", "", + "表示鉴权模式，取值可以为AK，StsToken，RamRoleArn，EcsRamRole，缺省值为空", + "specifies the authentication mode, the value can be AK，StsToken，RamRoleArn，EcsRamRole, default value is empty."}, + OptionECSRoleName: Option{"", "--ecs-role-name", "", OptionTypeString, "", "", + "表示角色名，主要用于EcsRamRole模式", + "specifies the authentication mode, primarily used in EcsRamRole mode."}, + OptionTokenTimeout: Option{"", "--token-timeout", "", OptionTypeInt64, "", "", + "表示token的有效时间，单位为秒, 缺省值为3600，主要用于RamRoleArn模式下的AssumeRole参数", + "specifies the valid time of a token, the unit is: s, default value is 3600, primarily used for AssumeRole parameters in RamRoleArn mode"}, + OptionRamRoleArn: Option{"", "--ram-role-arn", "", OptionTypeString, "", "", + "表示RAM角色的ARN，主要用于RamRoleArn模式", + "specifies the ARN of ram role, primarily used in RamRoleArn mode."}, + OptionRoleSessionName: Option{"", "--role-session-name", "", OptionTypeString, "", "", + "表示会话名字，主要用于RamRoleArn模式", + "specifies the session name, primarily used in RamRoleArn mode."}, + OptionReadTimeout: Option{"", "--read-timeout", "", OptionTypeInt64, "", "", + "表示客户端读超时的时间，单位为秒, 缺省值为1200", + "specifies the time that the client read timed out, the unit is: s, default value is 1200."}, + OptionConnectTimeout: Option{"", "--connect-timeout", "", OptionTypeInt64, "", "", + "表示客户端连接超时的时间，单位为秒, 缺省值为120", + "specifies the time that the client connection timed out, the unit is: s, default value is 120."}, + OptionSTSRegion: Option{"", "--sts-region", "", OptionTypeString, "", "", + "指定sts endpoint的地区，比如cn-shenzhen，其中，cn指代的是国家，shenzhen指代的是地区，用于构造sts endpoint，该选项缺省时，sts endpoint为中央endpoint，主要用于RamRoleArn模式", + "specifies the region of sts endpoint, such as cn-shenzhen, in this case, cn refers to the country and shenzhen refers to the region, to construct sts endpoint, when this option defaults, the sts endpoint is the central endpoint, primarily used in RamRoleArn mode."}, } func (T *Option) getHelp(language string) string { diff --git a/oss/lib/probe.go b/oss/lib/probe.go index fa58176ee..e7aacac2f 100644 --- a/oss/lib/probe.go +++ b/oss/lib/probe.go @@ -352,6 +352,14 @@ var probeCommand = ProbeCommand{ OptionLogLevel, OptionProbeItem, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/read_symlink.go b/oss/lib/read_symlink.go index d54bdd6ca..50aea1c94 100644 --- a/oss/lib/read_symlink.go +++ b/oss/lib/read_symlink.go @@ -116,6 +116,14 @@ var readSymlinkCommand = ReadSymlinkCommand{ OptionVersionId, OptionRequestPayer, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/request_payment.go b/oss/lib/request_payment.go index 5cce8597c..5466d6249 100644 --- a/oss/lib/request_payment.go +++ b/oss/lib/request_payment.go @@ -107,6 +107,14 @@ var requestPaymentCommand = RequestPaymentCommand{ OptionMethod, OptionLogLevel, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/request_payment_test.go b/oss/lib/request_payment_test.go index 608044b2e..98114eb0c 100644 --- a/oss/lib/request_payment_test.go +++ b/oss/lib/request_payment_test.go @@ -3,6 +3,7 @@ package lib import ( "io/ioutil" "os" + "time" . "gopkg.in/check.v1" ) @@ -34,6 +35,8 @@ func (s *OssutilCommandSuite) TestRequestPaymentPutSuccess(c *C) { _, err = cm.RunCommand("request-payment", requestArgs, options) c.Assert(err, IsNil) + time.Sleep(5 * time.Second) + // check strMethod = "get" requestArgs = []string{CloudURLToString(bucketName, "")} diff --git a/oss/lib/restore.go b/oss/lib/restore.go index 079afcc43..1408826d4 100644 --- a/oss/lib/restore.go +++ b/oss/lib/restore.go @@ -181,6 +181,14 @@ var restoreCommand = RestoreCommand{ OptionVersionId, OptionRequestPayer, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/revert_versioning.go b/oss/lib/revert_versioning.go index d4619d65d..519f7bc83 100644 --- a/oss/lib/revert_versioning.go +++ b/oss/lib/revert_versioning.go @@ -141,6 +141,14 @@ var revertCommand = RevertCommand{ OptionExclude, OptionEncodingType, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/rm.go b/oss/lib/rm.go index 6ddab4b45..bb8d15525 100644 --- a/oss/lib/rm.go +++ b/oss/lib/rm.go @@ -329,6 +329,14 @@ var removeCommand = RemoveCommand{ OptionAllversions, OptionRequestPayer, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/set_acl.go b/oss/lib/set_acl.go index 2fcdc548f..28c4a09c3 100644 --- a/oss/lib/set_acl.go +++ b/oss/lib/set_acl.go @@ -249,6 +249,14 @@ var setACLCommand = SetACLCommand{ OptionLogLevel, OptionVersionId, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/set_acl_test.go b/oss/lib/set_acl_test.go index 75ff51f89..5b00b735b 100644 --- a/oss/lib/set_acl_test.go +++ b/oss/lib/set_acl_test.go @@ -1202,7 +1202,7 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithInvalidIncExc(c *C) { func (s *OssutilCommandSuite) TestSetObjectAclWithVersion(c *C) { bucketName := bucketNamePrefix + "-set-alc-" + randLowStr(10) objectName := randStr(12) - + s.putBucket(bucketName, c) s.putBucketVersioning(bucketName, "enabled", c) @@ -1211,8 +1211,8 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithVersion(c *C) { s.createFile(uploadFileName, textBufferV1, c) s.putObject(bucketName, objectName, uploadFileName, c) s.setObjectACL(bucketName, objectName, "public-read", false, true, c) - objectStat := s.getStat(bucketName, objectName, c) - versionIdV1 := objectStat["X-Oss-Version-Id"] + objectStat := s.getStat(bucketName, objectName, c) + versionIdV1 := objectStat["X-Oss-Version-Id"] objectACLV1 := objectStat[StatACL] c.Assert(len(versionIdV1) > 0, Equals, true) c.Assert(objectACLV1, Equals, "public-read") @@ -1222,8 +1222,8 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithVersion(c *C) { s.createFile(uploadFileName, textBufferV2, c) s.putObject(bucketName, objectName, uploadFileName, c) s.setObjectACL(bucketName, objectName, "public-read-write", false, true, c) - objectStat = s.getStat(bucketName, objectName, c) - versionIdV2 := objectStat["X-Oss-Version-Id"] + objectStat = s.getStat(bucketName, objectName, c) + versionIdV2 := objectStat["X-Oss-Version-Id"] objectACLV2 := objectStat[StatACL] c.Assert(len(versionIdV2) > 0, Equals, true) c.Assert(objectACLV2, Equals, "public-read-write") @@ -1245,7 +1245,7 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithVersion(c *C) { c.Assert(err, IsNil) //get without versionId - objectStat = s.getStat(bucketName, objectName, c) + objectStat = s.getStat(bucketName, objectName, c) objectACLV2 = objectStat[StatACL] c.Assert(len(versionIdV2) > 0, Equals, true) c.Assert(objectACLV2, Equals, "public-read-write") @@ -1298,4 +1298,4 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithInvalidVersionArgs(c *C) { args = []string{CloudURLToString(bucketName, objectName), "private"} _, err = cm.RunCommand("set-acl", args, options) c.Assert(strings.Contains(err.Error(), "--version-id only work on single object"), Equals, true) -} +} \ No newline at end of file diff --git a/oss/lib/set_meta.go b/oss/lib/set_meta.go index b190a707f..dc1d5c567 100644 --- a/oss/lib/set_meta.go +++ b/oss/lib/set_meta.go @@ -301,6 +301,14 @@ var setMetaCommand = SetMetaCommand{ OptionLogLevel, OptionVersionId, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/signurl.go b/oss/lib/signurl.go index 8ac4029ce..f0875b546 100644 --- a/oss/lib/signurl.go +++ b/oss/lib/signurl.go @@ -128,6 +128,14 @@ var signURLCommand = SignurlCommand{ OptionTrafficLimit, OptionDisableEncodeSlash, OptionRequestPayer, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/stat.go b/oss/lib/stat.go index 072b23d60..a2d39d48d 100644 --- a/oss/lib/stat.go +++ b/oss/lib/stat.go @@ -113,6 +113,14 @@ var statCommand = StatCommand{ OptionVersionId, OptionRequestPayer, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/stat_test.go b/oss/lib/stat_test.go index b21d54f3a..463f384a7 100644 --- a/oss/lib/stat_test.go +++ b/oss/lib/stat_test.go @@ -253,6 +253,7 @@ func (s *OssutilCommandSuite) TestStatVersioning(c *C) { s.removeBucket(bucketName, true, c) } + func (s *OssutilCommandSuite) TestStatObjectWithPayer(c *C) { s.createFile(uploadFileName, content, c) bucketName := payerBucket diff --git a/oss/lib/sync.go b/oss/lib/sync.go index 27301c65c..7878293c8 100644 --- a/oss/lib/sync.go +++ b/oss/lib/sync.go @@ -413,6 +413,14 @@ var syncCommand = SyncCommand{ OptionDisableIgnoreError, OptionTagging, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, // The following options are only supported by sc command, not supported by cp command OptionDelete, diff --git a/oss/lib/user_qos.go b/oss/lib/user_qos.go index 2052e1e05..1f0741f7b 100644 --- a/oss/lib/user_qos.go +++ b/oss/lib/user_qos.go @@ -85,6 +85,14 @@ var userQosCommand = UserQosCommand{ OptionLogLevel, OptionMethod, OptionPassword, + OptionMode, + OptionECSRoleName, + OptionTokenTimeout, + OptionRamRoleArn, + OptionRoleSessionName, + OptionReadTimeout, + OptionConnectTimeout, + OptionSTSRegion, }, }, } diff --git a/oss/lib/util_sts.go b/oss/lib/util_sts.go new file mode 100644 index 000000000..d8b84dd9d --- /dev/null +++ b/oss/lib/util_sts.go @@ -0,0 +1,183 @@ +package lib + +import ( + "crypto/hmac" + "crypto/sha1" + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// Client sts client +type Client struct { + AccessKeyId string + AccessKeySecret string + RoleArn string + SessionName string +} + +// ServiceError sts service error +type ServiceError struct { + Code string + Message string + RequestId string + HostId string + RawMessage string + StatusCode int +} + +// Credentials the credentials obtained by AssumedRole, +// used for the peration of Alibaba Cloud service. +type Credentials struct { + AccessKeyId string + AccessKeySecret string + Expiration time.Time + SecurityToken string +} + +// AssumedRoleUser the user to AssumedRole +type AssumedRoleUser struct { + Arn string + AssumedRoleId string +} + +// Response the response of AssumeRole +type Response struct { + Credentials Credentials + AssumedRoleUser AssumedRoleUser + RequestId string +} + +// Error implement interface error +func (e *ServiceError) Error() string { + return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s", + e.StatusCode, e.Code, e.Message, e.RequestId) +} + +// NewClient New STS Client +func NewClient(accessKeyId, accessKeySecret, roleArn, sessionName string) *Client { + return &Client{ + AccessKeyId: accessKeyId, + AccessKeySecret: accessKeySecret, + RoleArn: roleArn, + SessionName: sessionName, + } +} + +const ( + // StsSignVersion sts sign version + StsSignVersion = "1.0" + // StsAPIVersion sts api version + StsAPIVersion = "2015-04-01" + + // // StsHost sts host + // StsHost = "https://sts.aliyuncs.com/" + + // TimeFormat time fomrat + TimeFormat = "2006-01-02T15:04:05Z" + // RespBodyFormat respone body format + RespBodyFormat = "JSON" + // PercentEncode '/' + PercentEncode = "%2F" + // HTTPGet http get method + HTTPGet = "GET" +) + +// StsHost sts host +var StsHost = "https://sts.aliyuncs.com/" + +// AssumeRole assume role +func (c *Client) AssumeRole(tokenTimeout uint, stsEndPoint string) (*Response, error) { + if stsEndPoint != "" { + StsHost = stsEndPoint + } + + url, err := c.generateSignedURL(tokenTimeout) + if err != nil { + return nil, err + } + + body, status, err := c.sendRequest(url) + if err != nil { + return nil, err + } + + return c.handleResponse(body, status) +} + +// Private function +func (c *Client) generateSignedURL(expiredTime uint) (string, error) { + randId := strings.ToUpper(randStr(24)) + + queryStr := "SignatureVersion=" + StsSignVersion + queryStr += "&Format=" + RespBodyFormat + queryStr += "&Timestamp=" + url.QueryEscape(time.Now().UTC().Format(TimeFormat)) + queryStr += "&RoleArn=" + url.QueryEscape(c.RoleArn) + queryStr += "&RoleSessionName=" + c.SessionName + queryStr += "&AccessKeyId=" + c.AccessKeyId + queryStr += "&SignatureMethod=HMAC-SHA1" + queryStr += "&Version=" + StsAPIVersion + queryStr += "&Action=AssumeRole" + queryStr += "&SignatureNonce=" + randId + queryStr += "&DurationSeconds=" + strconv.FormatUint((uint64)(expiredTime), 10) + + // Sort query string + queryParams, err := url.ParseQuery(queryStr) + if err != nil { + return "", err + } + result := queryParams.Encode() + + strToSign := HTTPGet + "&" + PercentEncode + "&" + url.QueryEscape(result) + + // Generate signature + hashSign := hmac.New(sha1.New, []byte(c.AccessKeySecret+"&")) + hashSign.Write([]byte(strToSign)) + signature := base64.StdEncoding.EncodeToString(hashSign.Sum(nil)) + + // Build url + assumeURL := StsHost + "?" + queryStr + "&Signature=" + url.QueryEscape(signature) + + return assumeURL, nil +} + +func (c *Client) sendRequest(url string) ([]byte, int, error) { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + resp, err := client.Get(url) + if err != nil { + return nil, -1, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + return body, resp.StatusCode, err +} + +func (c *Client) handleResponse(responseBody []byte, statusCode int) (*Response, error) { + if statusCode != http.StatusOK { + se := ServiceError{StatusCode: statusCode, RawMessage: string(responseBody)} + err := json.Unmarshal(responseBody, &se) + if err != nil { + return nil, err + } + return nil, &se + } + + resp := Response{} + err := json.Unmarshal(responseBody, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} diff --git a/oss/lib/util_sts_test.go b/oss/lib/util_sts_test.go new file mode 100644 index 000000000..66e1dd09b --- /dev/null +++ b/oss/lib/util_sts_test.go @@ -0,0 +1,128 @@ +package lib + +import ( + "fmt" + "log" + "os" + "time" + + . "gopkg.in/check.v1" +) + +type StsTestSuite struct { +} + +var _ = Suite(&StsTestSuite{}) + +// Run once when the suite starts running +func (s *StsTestSuite) SetUpSuite(c *C) { + accessKeyID = os.Getenv("OSS_TEST_STS_ID") + accessKeySecret = os.Getenv("OSS_TEST_STS_KEY") + stsARN = os.Getenv("OSS_TEST_STS_ARN") +} + +// Run after each test or benchmark starts running +func (s *StsTestSuite) TearDownSuite(c *C) { +} + +func (s *StsTestSuite) TestSendRequest(c *C) { + client := NewClient("", "", "", "") + _, _, err := client.sendRequest(StsHost) + c.Assert(err, IsNil) + + // negative + _, _, err = client.sendRequest("https//x.y.z.com") + c.Assert(err, NotNil) +} + +func (s *StsTestSuite) TestHandleResponse(c *C) { + client := NewClient("", "", "", "") + + body := "{\"RequestId\":\"784B99C1-895F-426C-8E1F-008955D418FB\"," + + "\"HostId\":\"sts.aliyuncs.com\"," + + "\"Code\":\"NoPermission\"," + + "\"Message\":\"Roles may not be assumed by root accounts.\"}" + resp, err := client.handleResponse([]byte(body), 400) + _, isSuc := err.(*ServiceError) + c.Assert(isSuc, Equals, true) + c.Assert(resp, IsNil) + + body = "{{}}" + resp, err = client.handleResponse([]byte(body), 400) + _, isSuc = err.(*ServiceError) + c.Assert(isSuc, Equals, false) + c.Assert(resp, IsNil) + + body = "{\"RequestId\":\"4AB89022-25A3-4427-84A5-4C7E72BD63BE\"}" + resp, err = client.handleResponse([]byte(body), 200) + c.Assert(err, IsNil) + c.Assert(resp, NotNil) + + body = "{{}}" + resp, err = client.handleResponse([]byte(body), 200) + _, isSuc = err.(*ServiceError) + c.Assert(isSuc, Equals, false) + c.Assert(resp, IsNil) +} + +func (s *StsTestSuite) TestAssumeRoleSuccess(c *C) { + now := time.Now() + client := NewClient(accessKeyID, accessKeySecret, stsARN, "sts_test") + + resp, err := client.AssumeRole(900, "") + if err != nil { + fmt.Println(err) + } else { + fmt.Println("success!") + } + c.Assert(err, IsNil) + + c.Assert(resp.RequestId, Not(Equals), "") + + c.Assert(resp.AssumedRoleUser.Arn, Not(Equals), "") + c.Assert(resp.AssumedRoleUser.AssumedRoleId, Not(Equals), "") + + c.Assert(resp.Credentials.AccessKeyId, Not(Equals), "") + c.Assert(resp.Credentials.AccessKeySecret, Not(Equals), "") + c.Assert(resp.Credentials.SecurityToken, Not(Equals), "") + c.Assert(resp.Credentials.Expiration.After(now), Equals, true) +} + +func (s *StsTestSuite) TestAssumeRoleNegative(c *C) { + // AccessKeyID invalid + client := NewClient("", accessKeySecret, stsARN, "sts_test") + resp, err := client.AssumeRole(900, "") + c.Assert(resp, IsNil) + c.Assert(err, NotNil) + log.Println("Error:", err) + + srvErr, isSuc := err.(*ServiceError) + c.Assert(isSuc, Equals, true) + c.Assert(srvErr.StatusCode, Equals, 400) + log.Println("ServiceError:", srvErr) + + // AccessKeySecret invalid + client = NewClient(accessKeyID, accessKeySecret+" ", stsARN, "sts_test") + resp, err = client.AssumeRole(900, "") + c.Assert(resp, IsNil) + c.Assert(err, NotNil) + + srvErr, isSuc = err.(*ServiceError) + c.Assert(isSuc, Equals, true) + c.Assert(srvErr.StatusCode, Equals, 400) + c.Assert(srvErr.Code, Equals, "SignatureDoesNotMatch") + log.Println("ServiceError:", srvErr) + + // SessionName invalid + client = NewClient(accessKeyID, accessKeySecret, stsARN, "x") + + resp, err = client.AssumeRole(900, "") + c.Assert(resp, IsNil) + c.Assert(err, NotNil) + + srvErr, isSuc = err.(*ServiceError) + c.Assert(isSuc, Equals, true) + c.Assert(srvErr.StatusCode, Equals, 400) + c.Assert(srvErr.Code, Equals, "InvalidParameter.RoleSessionName") + log.Println("ServiceError:", srvErr) +} From a91d3ad25263d51805a8240ee821398bd7b08a27 Mon Sep 17 00:00:00 2001 From: Jackson Tian Date: Fri, 2 Jul 2021 15:30:28 +0800 Subject: [PATCH 5/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f5872c5c6..733a6ce80 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ English | [简体中文](./README-CN.md)

Alibaba Cloud CLI

-Travis Build Status +Go build Status Appveyor Build Status codecov License