Merge remote-tracking branch 'upstream/main' into user/abhinavgupta/s…

…ample-custom-component

.gitignore

@@ -10,7 +10,7 @@ blobfuse2
 adlsgen1fuse
 venv/
 *.backup
-__debug_bin
+__debug_bin*
 .env
 *.prof
 cpplite/
@@ -21,3 +21,4 @@ lint.log
 azure-storage-fuse
 bfusemon
 test/scripts/dirIterate.go
+component/azstorage/logfile.txt

CHANGELOG.md

@@ -1,13 +1,19 @@
 ## 2.4.0 (Unreleased)
 **Bug Fixes**
 - [#1426](https://github.com/Azure/azure-storage-fuse/issues/1426) Read panic in block-cache due to boundary conditions.
-
+- Do not allow mount path and temp-cache path to be same when using block-cache.
+- Do not allow to mount with non-empty directory provided for disk persistence in block-cache.
+- Rename file was calling an additional getProperties call.
+
 **Features**
 - Added support for custom component via go plugin.
 
 **Other Changes**
 - Stream config will be converted to block-cache config implicitly and 'stream' component is no longer used from this release onwards.
 
+**Other Changes**
+- MSI login with object-id will not rely on azcli anymore.
+
 ## 2.3.2 (2024-09-03)
 **Bug Fixes**
 - Fixed the case where file creation using SAS on HNS accounts was returning back wrong error code.

NOTICE

@@ -3899,4 +3899,198 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
 
+
+
+
+
+****************************************************************************
+
+============================================================================
+>>> github.com/AzureAD/microsoft-authentication-extensions-for-go/cache
+==============================================================================
+
+    MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
+
+
+
+
+****************************************************************************
+
+============================================================================
+>>> github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache
+==============================================================================
+
+The MIT License (MIT)
+
+Copyright (c) Microsoft Corporation.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+
+
+****************************************************************************
+
+============================================================================
+>>> github.com/cespare/xxhash/v2
+==============================================================================
+
+Copyright (c) 2016 Caleb Spare
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+
+
+****************************************************************************
+
+============================================================================
+>>> github.com/dgryski/go-rendezvous
+==============================================================================
+
+The MIT License (MIT)
+
+Copyright (c) 2017-2020 Damian Gryski <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
+
+
+****************************************************************************
+
+============================================================================
+>>> github.com/keybase/go-keychain
+==============================================================================
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Keybase
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+
+
+
+****************************************************************************
+
+============================================================================
+>>> github.com/redis/go-redis/v9
+==============================================================================
+
+Copyright (c) 2013 The github.com/redis/go-redis Authors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------- END OF THIRD PARTY NOTICE --------------------------------

component/attr_cache/attr_cache.go

@@ -360,7 +360,6 @@ func (ac *AttrCache) RenameFile(options internal.RenameFileOptions) error {
 		ac.cacheLock.RLock()
 		defer ac.cacheLock.RUnlock()
 
-		// TODO: Can we just copy over the attributes from the source to the destination so we don't have to invalidate?
 		ac.deletePath(options.Src, time.Now())
 		ac.invalidatePath(options.Dst)
 	}

component/azstorage/azauthmsi.go

@@ -34,14 +34,6 @@
 package azstorage
 
 import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"os"
-	"os/exec"
-	"strings"
-
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
@@ -66,18 +58,22 @@ func (azmsi *azAuthMSI) getTokenCredential() (azcore.TokenCredential, error) {
 	}
 
 	if azmsi.config.ApplicationID != "" {
-		msiOpts.ID = (azidentity.ClientID)(azmsi.config.ApplicationID)
+		msiOpts.ID = azidentity.ClientID(azmsi.config.ApplicationID)
 	} else if azmsi.config.ResourceID != "" {
-		msiOpts.ID = (azidentity.ResourceID)(azmsi.config.ResourceID)
+		msiOpts.ID = azidentity.ResourceID(azmsi.config.ResourceID)
 	} else if azmsi.config.ObjectID != "" {
+		// Object id is supported by azidentity hence commenting the earlier code
+		msiOpts.ID = azidentity.ObjectID(azmsi.config.ObjectID)
+
 		// login using azcli
-		return azmsi.getTokenCredentialUsingCLI()
+		// return azmsi.getTokenCredentialUsingCLI()
 	}
 
 	cred, err := azidentity.NewManagedIdentityCredential(msiOpts)
 	return cred, err
 }
 
+/*
 func (azmsi *azAuthMSI) getTokenCredentialUsingCLI() (azcore.TokenCredential, error) {
 	command := "az login --identity --username " + azmsi.config.ObjectID
 
@@ -106,6 +102,7 @@ func (azmsi *azAuthMSI) getTokenCredentialUsingCLI() (azcore.TokenCredential, er
 	cred, err := azidentity.NewAzureCLICredential(nil)
 	return cred, err
 }
+*/
 
 type azAuthBlobMSI struct {
 	azAuthMSI

component/azstorage/block_blob.go

@@ -313,33 +313,27 @@ func (bb *BlockBlob) DeleteDirectory(name string) (err error) {
 }
 
 // RenameFile : Rename the file
+// Source file must exist in storage account before calling this method.
 func (bb *BlockBlob) RenameFile(source string, target string) error {
 	log.Trace("BlockBlob::RenameFile : %s -> %s", source, target)
 
 	blobClient := bb.Container.NewBlockBlobClient(filepath.Join(bb.Config.prefixPath, source))
 	newBlobClient := bb.Container.NewBlockBlobClient(filepath.Join(bb.Config.prefixPath, target))
 
-	_, err := blobClient.GetProperties(context.Background(), &blob.GetPropertiesOptions{
-		CPKInfo: bb.blobCPKOpt,
-	})
-	if err != nil {
-		serr := storeBlobErrToErr(err)
-		if serr == ErrFileNotFound {
-			log.Err("BlockBlob::RenameFile : %s does not exist", source)
-			return syscall.ENOENT
-		} else {
-			log.Err("BlockBlob::RenameFile : Failed to get blob properties for %s [%s]", source, err.Error())
-			return err
-		}
-	}
-
 	// not specifying source blob metadata, since passing empty metadata headers copies
 	// the source blob metadata to destination blob
 	startCopy, err := newBlobClient.StartCopyFromURL(context.Background(), blobClient.URL(), &blob.StartCopyFromURLOptions{
 		Tier: bb.Config.defaultTier,
 	})
 
 	if err != nil {
+		serr := storeBlobErrToErr(err)
+		if serr == ErrFileNotFound {
+			//Ideally this case doesn't hit as we are checking for the existence of src
+			//before making the call for RenameFile
+			log.Err("BlockBlob::RenameFile : Src Blob doesn't Exist %s [%s]", source, err.Error())
+			return syscall.ENOENT
+		}
 		log.Err("BlockBlob::RenameFile : Failed to start copy of file %s [%s]", source, err.Error())
 		return err
 	}
@@ -404,13 +398,26 @@ func (bb *BlockBlob) RenameDirectory(source string, target string) error {
 		}
 	}
 
-	err := bb.RenameFile(source, target)
-	// check if the marker blob for source directory does not exist but
-	// blobs were present in it, which were renamed earlier
-	if err == syscall.ENOENT && srcDirPresent {
-		err = nil
+	// To rename source marker blob check its properties before calling rename on it.
+	blobClient := bb.Container.NewBlockBlobClient(filepath.Join(bb.Config.prefixPath, source))
+	_, err := blobClient.GetProperties(context.Background(), &blob.GetPropertiesOptions{
+		CPKInfo: bb.blobCPKOpt,
+	})
+	if err != nil {
+		serr := storeBlobErrToErr(err)
+		if serr == ErrFileNotFound { //marker blob doesn't exist for the directory
+			if srcDirPresent { //Some files exist inside the directory
+				return nil
+			}
+			log.Err("BlockBlob::RenameDirectory : %s marker blob does not exist and Src Directory doesn't Exist", source)
+			return syscall.ENOENT
+		} else {
+			log.Err("BlockBlob::RenameDirectory : Failed to get source directory marker blob properties for %s [%s]", source, err.Error())
+			return err
+		}
 	}
-	return err
+
+	return bb.RenameFile(source, target)
 }
 
 func (bb *BlockBlob) getAttrUsingRest(name string) (attr *internal.ObjAttr, err error) {