option to add Cloud bucket to aistore without checking the bucket's a…

…ccessibility

* CLI: fix showing bucket props diff upon update
  (json tag `omit` in action)
* part two: docs, comments, and go.mod

Signed-off-by: Alex Aizman <[email protected]>

ais/proxy.go

@@ -1348,6 +1348,11 @@ func (p *proxy) _bcr(w http.ResponseWriter, r *http.Request, query url.Values, m
 	if bck.IsRemote() {
 		// (feature) add Cloud bucket to BMD, to further set its `Props.Extra`
 		// with alternative access profile and/or endpoint
+		// TODO:
+		// change bucket props - and the BMD meta-version - to have Flags int64 for
+		// the bits that'll include "renamed" (instead of the current `Props.Renamed`)
+		// and "added-with-no-head"; use the latter to synchronize Cloud props once
+		// connected
 		if cos.IsParseBool(query.Get(apc.QparamDontHeadRemote)) {
 			if !bck.IsCloud() {
 				p.writeErr(w, r, cmn.NewErrUnsupp("skip lookup for the", bck.Provider+":// bucket"))

cmd/cli/cli/bucket_hdlr.go

@@ -509,9 +509,19 @@ func showDiff(c *cli.Context, currProps, newProps *cmn.BucketProps) {
 		origKV = bckPropList(currProps, true)
 		newKV  = bckPropList(newProps, true)
 	)
-	for idx, prop := range newKV {
-		if origKV[idx].Value != prop.Value {
-			fmt.Fprintf(c.App.Writer, "%q set to: %q (was: %q)\n", prop.Name, prop.Value, origKV[idx].Value)
+	for _, np := range newKV {
+		var found bool
+		for _, op := range origKV {
+			if np.Name != op.Name {
+				continue
+			}
+			found = true
+			if np.Value != op.Value {
+				fmt.Fprintf(c.App.Writer, "%q set to: %q (was: %q)\n", np.Name, np.Value, op.Value)
+			}
+		}
+		if !found && np.Value != "" {
+			fmt.Fprintf(c.App.Writer, "%q set to: %q (was: n/a)\n", np.Name, np.Value)
 		}
 	}
 }

cmd/cli/cli/const.go

@@ -478,8 +478,8 @@ var (
 
 	addRemoteBucketWithNoLookupFlag = cli.BoolFlag{
 		Name: "skip-lookup",
-		Usage: "add Cloud bucket to aistore _unconditionally_ skipping check for accessibility\n" +
-			indent4 + "\t(note: usage must be limited to setting up bucket properties with alternative profile and/or endpoint)",
+		Usage: "add Cloud bucket to aistore without checking the bucket's accessibility and getting its Cloud properties\n" +
+			indent4 + "\t(usage must be limited to setting up bucket's aistore properties with alternative profile and/or endpoint)",
 	}
 
 	forceFlag = cli.BoolFlag{Name: "force,f", Usage: "force an action"}

cmd/cli/go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 // direct
 require (
-	github.com/NVIDIA/aistore v1.3.19-0.20230824193117-c572e42f2021
+	github.com/NVIDIA/aistore v1.3.19-0.20230824200641-cfe8329d6f64
 	github.com/fatih/color v1.14.1
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo v1.16.5

cmd/cli/go.sum

@@ -3,8 +3,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/aistore v1.3.19-0.20230824193117-c572e42f2021 h1:Nbi4J5BdrEgg9C2uDkIE0wta3iieT4XzPPFqjuVj8vw=
-github.com/NVIDIA/aistore v1.3.19-0.20230824193117-c572e42f2021/go.mod h1:tZvUalPk4wL/+5+5psJkZRHBqu3i2KV9g97HYyHvwc4=
+github.com/NVIDIA/aistore v1.3.19-0.20230824200641-cfe8329d6f64 h1:i/xALZmLd722rlncwOeVYXzrlHXpSQn5dY4r+GvVlOQ=
+github.com/NVIDIA/aistore v1.3.19-0.20230824200641-cfe8329d6f64/go.mod h1:tZvUalPk4wL/+5+5psJkZRHBqu3i2KV9g97HYyHvwc4=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=

docs/cli/aws_profile_endpoint.md

@@ -26,6 +26,7 @@ For supported backends (that include, but are not limited, to AWS S3), see also:
 - [Viewing vendor-specific properties](#viewing-vendor-specific-properties)
 - [Environment variables](#environment-variables)
 - [Setting profile with alternative access/secret keys and/or region](#setting-profile-with-alternative-accesssecret-keys-andor-region)
+- [When bucket does not exist](#when-bucket-does-not-exist)
 - [Configuring custom AWS S3 endpoint](#configuring-custom-aws-s3-endpoint)
 
 ## Viewing vendor-specific properties
@@ -95,6 +96,29 @@ extra.aws.profile           prod
 
 From this point on, all calls to read, write, list `s3://abc` and get/set its properties will use AWS "prod" profile (see above).
 
+## When bucket does not exist
+
+But what if we need to set alternative profile (with alternative access and secret keys) on a bucket that does not yet exist in the cluster?
+
+That must be a fairly common situation, and the way to resolve it is to use `--skip-lookup` option:
+
+```console
+$ ais create --help
+...
+OPTIONS:
+   --props value   bucket properties, e.g. --props="mirror.enabled=true mirror.copies=4 checksum.type=md5"
+   --skip-lookup   add Cloud bucket to aistore without checking the bucket's accessibility and getting its Cloud properties
+                   (usage must be limited to setting up bucket's aistore properties with alternative profile and/or endpoint)
+
+
+$ ais create s3://abc --skip-lookup
+"s3://abc" created
+```
+
+Once this is done (**), we simply go ahead and run `ais bucket props set s3://abc extra.aws.profile` (as shown above). Assuming, the updated profile contains correct access keys, the bucket will then be fully available for reading, writing, listing, and all the rest operations.
+
+> (**) `ais create` command results in adding the bucket to aistore `BMD` - a protected, versioned, and replicated bucket metadata that is further used to update properties of any bucket in the cluster, including certainly the one that we have just added.
+
 ## Configuring custom AWS S3 endpoint
 
 When a bucket is hosted by an S3 compliant backend (such as, e.g., minio), we may want to specify an alternative S3 endpoint,