diff --git a/client/fed_test.go b/client/fed_test.go index 5871ec407..094c2f0d5 100644 --- a/client/fed_test.go +++ b/client/fed_test.go @@ -48,6 +48,7 @@ import ( "github.com/pelicanplatform/pelican/test_utils" "github.com/pelicanplatform/pelican/token" "github.com/pelicanplatform/pelican/token_scopes" + "github.com/pelicanplatform/pelican/utils" ) var ( @@ -144,16 +145,48 @@ func TestGetAndPutAuth(t *testing.T) { // Upload the file with PUT transferResultsUpload, err := client.DoPut(fed.Ctx, tempFile.Name(), uploadURL, false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) - } + assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) // Download that same file with GET transferResultsDownload, err := client.DoGet(fed.Ctx, uploadURL, t.TempDir(), false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) - } + assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) + } + }) + + t.Run("testPelicanObjectPutAndGetWithQueryAndDestDir", func(t *testing.T) { + oldPref, err := config.SetPreferredPrefix(config.PelicanPrefix) + defer func() { + _, err := config.SetPreferredPrefix(oldPref) + require.NoError(t, err) + }() + assert.NoError(t, err) + viper.Set(param.Federation_DiscoveryUrl.GetName(), fmt.Sprintf("%s://%s:%s", "https", param.Server_Hostname.GetString(), strconv.Itoa(param.Server_WebPort.GetInt()))) + + // Set path for object to upload/download + for _, export := range fed.Exports { + tempPath := tempFile.Name() + fileName := filepath.Base(tempPath) + uploadURL := fmt.Sprintf("%s/%s/%s", + export.FederationPrefix, "osdf_osdf", fileName) + + uploadURL, err := utils.UrlWithFederation(uploadURL) + assert.NoError(t, err) + + // Upload the file with PUT + transferResultsUpload, err := client.DoPut(fed.Ctx, tempFile.Name(), uploadURL, false, client.WithTokenLocation(tempToken.Name())) + assert.NoError(t, err) + assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) + + queryURL := uploadURL + "?directread" + tempDir := t.TempDir() + // Download that same file with GET + transferResultsDownload, err := client.DoGet(fed.Ctx, queryURL, tempDir, false, client.WithTokenLocation(tempToken.Name())) + assert.NoError(t, err) + assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) + stats, err := os.Stat(filepath.Join(tempDir, fileName)) + assert.NoError(t, err) + assert.NotNil(t, stats) } }) @@ -176,16 +209,12 @@ func TestGetAndPutAuth(t *testing.T) { // Upload the file with PUT transferResultsUpload, err := client.DoPut(fed.Ctx, tempFile.Name(), uploadURL, false, client.WithToken(tmpTkn)) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) - } + assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) // Download that same file with GET transferResultsDownload, err := client.DoGet(fed.Ctx, uploadURL, t.TempDir(), false, client.WithToken(tmpTkn)) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) - } + assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) } }) @@ -208,16 +237,12 @@ func TestGetAndPutAuth(t *testing.T) { // Upload the file with PUT transferResultsUpload, err := client.DoPut(fed.Ctx, tempFile.Name(), uploadURL, false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) - } + assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) // Download that same file with GET transferResultsDownload, err := client.DoGet(fed.Ctx, uploadURL, t.TempDir(), false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) - } + assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) } }) @@ -248,16 +273,12 @@ func TestGetAndPutAuth(t *testing.T) { // Upload the file with PUT transferResultsUpload, err := client.DoPut(fed.Ctx, tempFile.Name(), uploadURL, false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) - } + assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) // Download that same file with GET transferResultsDownload, err := client.DoGet(fed.Ctx, uploadURL, t.TempDir(), false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) - } + assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) } }) t.Cleanup(func() { @@ -307,19 +328,52 @@ func TestCopyAuth(t *testing.T) { uploadURL := fmt.Sprintf("pelican://%s:%s%s/%s/%s", param.Server_Hostname.GetString(), strconv.Itoa(param.Server_WebPort.GetInt()), export.FederationPrefix, "osdf_osdf", fileName) - // Upload the file with PUT + // Upload the file with COPY transferResultsUpload, err := client.DoCopy(fed.Ctx, tempFile.Name(), uploadURL, false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, int64(17), transferResultsUpload[0].TransferredBytes) - } + assert.Equal(t, int64(17), transferResultsUpload[0].TransferredBytes) - // Download that same file with GET + // Download that same file with COPY transferResultsDownload, err := client.DoCopy(fed.Ctx, uploadURL, t.TempDir(), false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, int64(17), transferResultsDownload[0].TransferredBytes) - } + assert.Equal(t, int64(17), transferResultsDownload[0].TransferredBytes) + } + }) + + t.Run("testPelicanObjectCopyWithQueryAndDestDir", func(t *testing.T) { + oldPref, err := config.SetPreferredPrefix(config.PelicanPrefix) + assert.NoError(t, err) + defer func() { + _, err := config.SetPreferredPrefix(oldPref) + require.NoError(t, err) + }() + + viper.Set(param.Federation_DiscoveryUrl.GetName(), fmt.Sprintf("%s://%s:%s", "https", param.Server_Hostname.GetString(), strconv.Itoa(param.Server_WebPort.GetInt()))) + + // Set path for object to upload/download + for _, export := range fed.Exports { + tempPath := tempFile.Name() + fileName := filepath.Base(tempPath) + uploadURL := fmt.Sprintf("%s/%s/%s", + export.FederationPrefix, "osdf_osdf", fileName) + + uploadURL, err := utils.UrlWithFederation(uploadURL) + assert.NoError(t, err) + + // Upload the file with COPY + transferResultsUpload, err := client.DoCopy(fed.Ctx, tempFile.Name(), uploadURL, false, client.WithTokenLocation(tempToken.Name())) + assert.NoError(t, err) + assert.Equal(t, int64(17), transferResultsUpload[0].TransferredBytes) + + queryURL := uploadURL + "?directread" + tempDir := t.TempDir() + // Download that same file with COPY + transferResultsDownload, err := client.DoCopy(fed.Ctx, queryURL, tempDir, false, client.WithTokenLocation(tempToken.Name())) + assert.NoError(t, err) + assert.Equal(t, int64(17), transferResultsDownload[0].TransferredBytes) + stats, err := os.Stat(filepath.Join(tempDir, fileName)) + assert.NoError(t, err) + assert.NotNil(t, stats) } }) @@ -342,16 +396,12 @@ func TestCopyAuth(t *testing.T) { // Upload the file with PUT transferResultsUpload, err := client.DoCopy(fed.Ctx, tempFile.Name(), uploadURL, false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) - } + assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) // Download that same file with GET transferResultsDownload, err := client.DoCopy(fed.Ctx, uploadURL, t.TempDir(), false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) - } + assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) } }) @@ -382,16 +432,13 @@ func TestCopyAuth(t *testing.T) { // Upload the file with PUT transferResultsUpload, err := client.DoCopy(fed.Ctx, tempFile.Name(), uploadURL, false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) - } + assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) // Download that same file with GET transferResultsDownload, err := client.DoCopy(fed.Ctx, uploadURL, t.TempDir(), false, client.WithTokenLocation(tempToken.Name())) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) - } + assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) + } }) t.Cleanup(func() { @@ -431,9 +478,7 @@ func TestGetPublicRead(t *testing.T) { // Download the file with GET. Shouldn't need a token to succeed transferResults, err := client.DoGet(fed.Ctx, uploadURL, t.TempDir(), false) assert.NoError(t, err) - if err == nil { - assert.Equal(t, transferResults[0].TransferredBytes, int64(17)) - } + assert.Equal(t, transferResults[0].TransferredBytes, int64(17)) } }) t.Cleanup(func() { @@ -543,10 +588,8 @@ func TestObjectStat(t *testing.T) { // Stat the file statInfo, err := client.DoStat(fed.Ctx, statUrl) assert.NoError(t, err) - if err == nil { - assert.Equal(t, int64(17), int64(statInfo.Size)) - assert.Equal(t, fmt.Sprintf("%s/%s", fed.Exports[0].FederationPrefix, fileName), statInfo.Name) - } + assert.Equal(t, int64(17), int64(statInfo.Size)) + assert.Equal(t, fmt.Sprintf("%s/%s", fed.Exports[0].FederationPrefix, fileName), statInfo.Name) }) // Ensure stat fails if it does not recognize the url scheme diff --git a/cmd/object_copy.go b/cmd/object_copy.go index e5678017e..344194dd2 100644 --- a/cmd/object_copy.go +++ b/cmd/object_copy.go @@ -202,6 +202,11 @@ func copyMain(cmd *cobra.Command, args []string) { lastSrc := "" for _, src := range source { + src, result = utils.UrlWithFederation(src) + if result != nil { + lastSrc = src + break + } isRecursive, _ := cmd.Flags().GetBool("recursive") _, result = client.DoCopy(ctx, src, dest, isRecursive, client.WithCallback(pb.callback), client.WithTokenLocation(tokenLocation), client.WithCaches(caches...)) if result != nil { diff --git a/cmd/object_get.go b/cmd/object_get.go index 98d2783c1..fba902cb5 100644 --- a/cmd/object_get.go +++ b/cmd/object_get.go @@ -121,6 +121,11 @@ func getMain(cmd *cobra.Command, args []string) { lastSrc := "" for _, src := range source { + src, result = utils.UrlWithFederation(src) + if result != nil { + lastSrc = src + break + } isRecursive, _ := cmd.Flags().GetBool("recursive") _, result = client.DoGet(ctx, src, dest, isRecursive, client.WithCallback(pb.callback), client.WithTokenLocation(tokenLocation), client.WithCaches(caches...)) if result != nil { diff --git a/utils/client_test.go b/utils/client_test.go index 20efd4c21..38bef2ca4 100644 --- a/utils/client_test.go +++ b/utils/client_test.go @@ -19,10 +19,14 @@ package utils import ( + "fmt" "net/url" "testing" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" + + "github.com/pelicanplatform/pelican/param" ) // Test the functionality of CheckValidQuery and all its edge cases @@ -238,3 +242,47 @@ func TestExtractVersionAndServiceFromUserAgent(t *testing.T) { assert.Equal(t, 0, len(service)) }) } + +func TestUrlWithFederation(t *testing.T) { + viper.Reset() + defer viper.Reset() + pelUrl := "pelican://somefederation.org/namespace/test.txt" + + t.Run("testNoFederation", func(t *testing.T) { + str, err := UrlWithFederation(pelUrl) + assert.NoError(t, err) + assert.Equal(t, pelUrl, str) + }) + + t.Run("testFederationNoHost", func(t *testing.T) { + viper.Set(param.Federation_DiscoveryUrl.GetName(), "somefederation.org") + namespaceOnly := "/namespace/test.txt" + str, err := UrlWithFederation(namespaceOnly) + assert.NoError(t, err) + assert.Equal(t, pelUrl, str) + }) + + t.Run("testFederationWithFedHost", func(t *testing.T) { + viper.Set(param.Federation_DiscoveryUrl.GetName(), "https://somefederation.org") + namespaceOnly := "/namespace/test.txt" + str, err := UrlWithFederation(namespaceOnly) + assert.NoError(t, err) + assert.Equal(t, pelUrl, str) + }) + + t.Run("testFederationWithPathComponent", func(t *testing.T) { + viper.Set(param.Federation_DiscoveryUrl.GetName(), "somefederation.org/path") + namespaceOnly := "/namespace/test.txt" + _, err := UrlWithFederation(namespaceOnly) + assert.Error(t, err) + assert.EqualError(t, err, fmt.Sprintf("provided federation url %s has a path component", param.Federation_DiscoveryUrl.GetString())) + }) + + t.Run("testFederationPathComponentWithHost", func(t *testing.T) { + viper.Set(param.Federation_DiscoveryUrl.GetName(), "https://somefederation.org/path") + namespaceOnly := "/namespace/test.txt" + _, err := UrlWithFederation(namespaceOnly) + assert.Error(t, err) + assert.EqualError(t, err, fmt.Sprintf("provided federation url %s has a path component", param.Federation_DiscoveryUrl.GetString())) + }) +} diff --git a/utils/utils.go b/utils/utils.go index 0628e78d4..a2d3464f1 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -19,6 +19,7 @@ package utils import ( + "fmt" "net" "net/url" "regexp" @@ -29,6 +30,8 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/text/cases" "golang.org/x/text/language" + + "github.com/pelicanplatform/pelican/param" ) // snakeCaseToCamelCase converts a snake case string to camel case. @@ -127,3 +130,37 @@ func ExtractVersionAndServiceFromUserAgent(userAgent string) (reqVer, service st service = (strings.Split(userAgentSplit[0], "-"))[1] return reqVer, service } + +// Given a remote url with no host (a namespace path, essentially), then if the Federation.DiscoveryURL is +// set (either via a yaml file or via the command line flags), create a full pelican protocol URL by combining +// the two +func UrlWithFederation(remoteUrl string) (string, error) { + if param.Federation_DiscoveryUrl.IsSet() { + parsedUrl, err := url.Parse(remoteUrl) + if err != nil { + newErr := errors.New(fmt.Sprintf("error parsing source url: %s", err)) + return remoteUrl, newErr + } + if parsedUrl.Host != "" { + return remoteUrl, nil + } + parsedDiscUrl, err := url.Parse(param.Federation_DiscoveryUrl.GetString()) + if err != nil { + newErr := errors.New(fmt.Sprintf("error parsing discovery url: %s", err)) + return remoteUrl, newErr + } + if parsedDiscUrl.Scheme == "" { + parsedDiscUrl.Scheme = "https" + updatedDiscString := parsedDiscUrl.String() + parsedDiscUrl, _ = url.Parse(updatedDiscString) + } + if parsedDiscUrl.Path != "" { + newErr := errors.New(fmt.Sprintf("provided federation url %s has a path component", param.Federation_DiscoveryUrl.GetString())) + return remoteUrl, newErr + } + parsedUrl.Host = parsedDiscUrl.Host + parsedUrl.Scheme = "pelican" + return parsedUrl.String(), nil + } + return remoteUrl, nil +}