diff --git a/ais/proxy.go b/ais/proxy.go index f1a6d8d1c1..4798026a79 100644 --- a/ais/proxy.go +++ b/ais/proxy.go @@ -1348,6 +1348,11 @@ func (p *proxy) _bcr(w http.ResponseWriter, r *http.Request, query url.Values, m if bck.IsRemote() { // (feature) add Cloud bucket to BMD, to further set its `Props.Extra` // with alternative access profile and/or endpoint + // TODO: + // change bucket props - and the BMD meta-version - to have Flags int64 for + // the bits that'll include "renamed" (instead of the current `Props.Renamed`) + // and "added-with-no-head"; use the latter to synchronize Cloud props once + // connected if cos.IsParseBool(query.Get(apc.QparamDontHeadRemote)) { if !bck.IsCloud() { p.writeErr(w, r, cmn.NewErrUnsupp("skip lookup for the", bck.Provider+":// bucket")) diff --git a/cmd/cli/cli/bucket_hdlr.go b/cmd/cli/cli/bucket_hdlr.go index fc29ec96c8..d14845fd17 100644 --- a/cmd/cli/cli/bucket_hdlr.go +++ b/cmd/cli/cli/bucket_hdlr.go @@ -509,9 +509,19 @@ func showDiff(c *cli.Context, currProps, newProps *cmn.BucketProps) { origKV = bckPropList(currProps, true) newKV = bckPropList(newProps, true) ) - for idx, prop := range newKV { - if origKV[idx].Value != prop.Value { - fmt.Fprintf(c.App.Writer, "%q set to: %q (was: %q)\n", prop.Name, prop.Value, origKV[idx].Value) + for _, np := range newKV { + var found bool + for _, op := range origKV { + if np.Name != op.Name { + continue + } + found = true + if np.Value != op.Value { + fmt.Fprintf(c.App.Writer, "%q set to: %q (was: %q)\n", np.Name, np.Value, op.Value) + } + } + if !found && np.Value != "" { + fmt.Fprintf(c.App.Writer, "%q set to: %q (was: n/a)\n", np.Name, np.Value) } } } diff --git a/cmd/cli/cli/const.go b/cmd/cli/cli/const.go index f6413f6f25..6df2fc51cd 100644 --- a/cmd/cli/cli/const.go +++ b/cmd/cli/cli/const.go @@ -478,8 +478,8 @@ var ( addRemoteBucketWithNoLookupFlag = cli.BoolFlag{ Name: "skip-lookup", - Usage: "add Cloud bucket to aistore _unconditionally_ skipping check for accessibility\n" + - indent4 + "\t(note: usage must be limited to setting up bucket properties with alternative profile and/or endpoint)", + Usage: "add Cloud bucket to aistore without checking the bucket's accessibility and getting its Cloud properties\n" + + indent4 + "\t(usage must be limited to setting up bucket's aistore properties with alternative profile and/or endpoint)", } forceFlag = cli.BoolFlag{Name: "force,f", Usage: "force an action"} diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod index ddbffd81d4..bb3c20ebcc 100644 --- a/cmd/cli/go.mod +++ b/cmd/cli/go.mod @@ -4,7 +4,7 @@ go 1.20 // direct require ( - github.com/NVIDIA/aistore v1.3.19-0.20230824193117-c572e42f2021 + github.com/NVIDIA/aistore v1.3.19-0.20230824200641-cfe8329d6f64 github.com/fatih/color v1.14.1 github.com/json-iterator/go v1.1.12 github.com/onsi/ginkgo v1.16.5 diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum index 3a7456a88f..2b2005ced2 100644 --- a/cmd/cli/go.sum +++ b/cmd/cli/go.sum @@ -3,8 +3,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/NVIDIA/aistore v1.3.19-0.20230824193117-c572e42f2021 h1:Nbi4J5BdrEgg9C2uDkIE0wta3iieT4XzPPFqjuVj8vw= -github.com/NVIDIA/aistore v1.3.19-0.20230824193117-c572e42f2021/go.mod h1:tZvUalPk4wL/+5+5psJkZRHBqu3i2KV9g97HYyHvwc4= +github.com/NVIDIA/aistore v1.3.19-0.20230824200641-cfe8329d6f64 h1:i/xALZmLd722rlncwOeVYXzrlHXpSQn5dY4r+GvVlOQ= +github.com/NVIDIA/aistore v1.3.19-0.20230824200641-cfe8329d6f64/go.mod h1:tZvUalPk4wL/+5+5psJkZRHBqu3i2KV9g97HYyHvwc4= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= diff --git a/docs/cli/aws_profile_endpoint.md b/docs/cli/aws_profile_endpoint.md index 4c7d7f8b4b..103fa6c6d0 100644 --- a/docs/cli/aws_profile_endpoint.md +++ b/docs/cli/aws_profile_endpoint.md @@ -26,6 +26,7 @@ For supported backends (that include, but are not limited, to AWS S3), see also: - [Viewing vendor-specific properties](#viewing-vendor-specific-properties) - [Environment variables](#environment-variables) - [Setting profile with alternative access/secret keys and/or region](#setting-profile-with-alternative-accesssecret-keys-andor-region) +- [When bucket does not exist](#when-bucket-does-not-exist) - [Configuring custom AWS S3 endpoint](#configuring-custom-aws-s3-endpoint) ## Viewing vendor-specific properties @@ -95,6 +96,29 @@ extra.aws.profile prod From this point on, all calls to read, write, list `s3://abc` and get/set its properties will use AWS "prod" profile (see above). +## When bucket does not exist + +But what if we need to set alternative profile (with alternative access and secret keys) on a bucket that does not yet exist in the cluster? + +That must be a fairly common situation, and the way to resolve it is to use `--skip-lookup` option: + +```console +$ ais create --help +... +OPTIONS: + --props value bucket properties, e.g. --props="mirror.enabled=true mirror.copies=4 checksum.type=md5" + --skip-lookup add Cloud bucket to aistore without checking the bucket's accessibility and getting its Cloud properties + (usage must be limited to setting up bucket's aistore properties with alternative profile and/or endpoint) + + +$ ais create s3://abc --skip-lookup +"s3://abc" created +``` + +Once this is done (**), we simply go ahead and run `ais bucket props set s3://abc extra.aws.profile` (as shown above). Assuming, the updated profile contains correct access keys, the bucket will then be fully available for reading, writing, listing, and all the rest operations. + +> (**) `ais create` command results in adding the bucket to aistore `BMD` - a protected, versioned, and replicated bucket metadata that is further used to update properties of any bucket in the cluster, including certainly the one that we have just added. + ## Configuring custom AWS S3 endpoint When a bucket is hosted by an S3 compliant backend (such as, e.g., minio), we may want to specify an alternative S3 endpoint,