Skip to content

Commit

Permalink
Finished: remove use of metadata in client
Browse files Browse the repository at this point in the history 
Changes include:
- Addressing comments on PR
- Making unit tests for new functions
- Fix previous typo in "discovery"
- Added ttl cache for federations
- Fixed merge conflicts
  • Loading branch information
@joereuss12
joereuss12 committed Feb 19, 2024
1 parent 484aa68 commit 144ec34
Show file tree
Hide file tree
Showing 6 changed files with 369 additions and 188 deletions.
16 changes: 8 additions & 8 deletions client/handle_http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ func GenerateTransferDetailsUsingCache(cache CacheInterface, opts TransferDetail
return nil
}

func download_http(sourceUrl *url.URL, destination string, metadata UrlMetadata, payload *payloadStruct, namespace namespaces.Namespace, recursive bool, tokenName string) (transferResults []TransferResults, err error) {
func download_http(pelicanURL PelicanURL, destination string, payload *payloadStruct, namespace namespaces.Namespace, recursive bool, tokenName string) (transferResults []TransferResults, err error) {
// First, create a handler for any panics that occur
defer func() {
if r := recover(); r != nil {
Expand All @@ -288,16 +288,16 @@ func download_http(sourceUrl *url.URL, destination string, metadata UrlMetadata,
}
}()

packOption := sourceUrl.Query().Get("pack")
packOption := pelicanURL.objectUrl.Query().Get("pack")
if packOption != "" {
log.Debugln("Will use unpack option value", packOption)
}
sourceUrl = &url.URL{Path: sourceUrl.Path}
pelicanURL.objectUrl = &url.URL{Path: pelicanURL.objectUrl.Path}

var token string
if namespace.UseTokenOnRead {
var err error
token, err = getToken(sourceUrl, namespace, false, tokenName)
token, err = getToken(pelicanURL.objectUrl, namespace, false, tokenName)
if err != nil {
log.Errorln("Failed to get token though required to read from this namespace:", err)
return nil, err
Expand All @@ -307,7 +307,7 @@ func download_http(sourceUrl *url.URL, destination string, metadata UrlMetadata,
// Check the env var "USE_OSDF_DIRECTOR" and decide if ordered caches should come from director
var transfers []TransferDetails
var files []string
directorUrl := metadata.directorUrl
directorUrl := pelicanURL.directorUrl
closestNamespaceCaches, err := GetCachesFromNamespace(namespace, directorUrl != "")
if err != nil {
log.Errorln("Failed to get namespaced caches (treated as non-fatal):", err)
Expand All @@ -324,13 +324,13 @@ func download_http(sourceUrl *url.URL, destination string, metadata UrlMetadata,

if recursive {
var err error
files, err = walkDavDir(sourceUrl, namespace, token, "", false)
files, err = walkDavDir(pelicanURL.objectUrl, namespace, token, "", false)
if err != nil {
log.Errorln("Error from walkDavDir", err)
return nil, err
}
} else {
files = append(files, sourceUrl.Path)
files = append(files, pelicanURL.objectUrl.Path)
}

for _, cache := range closestNamespaceCaches[:cachesToTry] {
Expand Down Expand Up @@ -362,7 +362,7 @@ func download_http(sourceUrl *url.URL, destination string, metadata UrlMetadata,
// Start the workers
for i := 1; i <= 5; i++ {
wg.Add(1)
go startDownloadWorker(sourceUrl.Path, destination, token, transfers, payload, &wg, workChan, results)
go startDownloadWorker(pelicanURL.objectUrl.Path, destination, token, transfers, payload, &wg, workChan, results)
}

// For each file, send it to the worker
Expand Down
76 changes: 38 additions & 38 deletions client/handle_http_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,7 @@ func TestObjectCopyAuth(t *testing.T) {
assert.NoError(t, err, "Error writing to temp token file")
tempToken.Close()
// Disable progress bars to not reuse the same mpb instance
viper.Set("Logging.DisableProgressBars", true)
ObjectClientOptions.ProgressBars = false

// This tests pelican object get/put with a pelican:// url
t.Run("testPelicanObjectCopyWithPelicanUrl", func(t *testing.T) {
Expand All @@ -581,7 +581,7 @@ func TestObjectCopyAuth(t *testing.T) {
fileName := filepath.Base(tempPath)
hostname, err := os.Hostname()
assert.NoError(t, err)
uploadURL := "pelican://"+ hostname + ":8444/test/" + fileName
uploadURL := "pelican://" + hostname + ":8444/test/" + fileName

// Upload the file with PUT
ObjectClientOptions.Token = tempToken.Name()
Expand Down Expand Up @@ -611,13 +611,13 @@ func TestObjectCopyAuth(t *testing.T) {
assert.NoError(t, err)

// For OSDF url's, we don't want to rely on osdf metadata to be running therefore, just ensure we get correct metadata for the url:
metadata, err := getUrlMetadata(uploadURL, "osdf")
pelicanURL, err := newPelicanURL(uploadURL, "osdf")
assert.NoError(t, err)

// Check valid metadata:
assert.Equal(t, "https://osdf-director.osg-htc.org", metadata.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", metadata.registryUrl)
assert.Equal(t, "osg-htc.org", metadata.discoveryUrl)
assert.Equal(t, "https://osdf-director.osg-htc.org", pelicanURL.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", pelicanURL.registryUrl)
assert.Equal(t, "osg-htc.org", pelicanURL.discoveryUrl)
})

// This tests osdf object copy with a pelican:// url
Expand All @@ -628,7 +628,7 @@ func TestObjectCopyAuth(t *testing.T) {
fileName := filepath.Base(tempPath)
hostname, err := os.Hostname()
assert.NoError(t, err)
uploadURL := "pelican://"+ hostname + ":8444/test/" + fileName
uploadURL := "pelican://" + hostname + ":8444/test/" + fileName

// Upload the file with PUT
ObjectClientOptions.Token = tempToken.Name()
Expand All @@ -648,7 +648,7 @@ func TestObjectCopyAuth(t *testing.T) {
})

// This tests osdf object copy with an osdf url
// NOTE: this test MUST be last since we need to reset a good amount of config values to get osdf defaults
// NOTE: this test MUST be last since we need to reset a good amount of config values to get osdf defaults as well as re-init config and client
t.Run("testOsdfObjectCopyWithOSDFUrl", func(t *testing.T) {
viper.Reset()
config.SetPreferredPrefix("OSDF")
Expand All @@ -662,13 +662,13 @@ func TestObjectCopyAuth(t *testing.T) {
assert.NoError(t, err)

// For OSDF url's, we don't want to rely on osdf metadata to be running therefore, just ensure we get correct metadata for the url:
metadata, err := getUrlMetadata(uploadURL, "osdf")
pelicanURL, err := newPelicanURL(uploadURL, "osdf")
assert.NoError(t, err)

// Check valid metadata:
assert.Equal(t, "https://osdf-director.osg-htc.org", metadata.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", metadata.registryUrl)
assert.Equal(t, "osg-htc.org", metadata.discoveryUrl)
assert.Equal(t, "https://osdf-director.osg-htc.org", pelicanURL.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", pelicanURL.registryUrl)
assert.Equal(t, "osg-htc.org", pelicanURL.discoveryUrl)
})
}

Expand Down Expand Up @@ -720,7 +720,7 @@ func TestGetAndPutAuth(t *testing.T) {
assert.NoError(t, err, "Error writing to temp token file")
tempToken.Close()
// Disable progress bars to not reuse the same mpb instance
viper.Set("Logging.DisableProgressBars", true)
ObjectClientOptions.ProgressBars = false

// This tests pelican object get/put with a pelican:// url
t.Run("testPelicanObjectPutAndGetWithPelicanUrl", func(t *testing.T) {
Expand All @@ -730,7 +730,7 @@ func TestGetAndPutAuth(t *testing.T) {
fileName := filepath.Base(tempPath)
hostname, err := os.Hostname()
assert.NoError(t, err)
uploadURL := "pelican://"+ hostname + ":8444/test/" + fileName
uploadURL := "pelican://" + hostname + ":8444/test/" + fileName

// Upload the file with PUT
ObjectClientOptions.Token = tempToken.Name()
Expand Down Expand Up @@ -760,13 +760,13 @@ func TestGetAndPutAuth(t *testing.T) {
assert.NoError(t, err)

// For OSDF url's, we don't want to rely on osdf metadata to be running therefore, just ensure we get correct metadata for the url:
metadata, err := getUrlMetadata(uploadURL, "osdf")
pelicanURL, err := newPelicanURL(uploadURL, "osdf")
assert.NoError(t, err)

// Check valid metadata:
assert.Equal(t, "https://osdf-director.osg-htc.org", metadata.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", metadata.registryUrl)
assert.Equal(t, "osg-htc.org", metadata.discoveryUrl)
assert.Equal(t, "https://osdf-director.osg-htc.org", pelicanURL.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", pelicanURL.registryUrl)
assert.Equal(t, "osg-htc.org", pelicanURL.discoveryUrl)
})

// This tests object get/put with a pelican:// url
Expand All @@ -777,7 +777,7 @@ func TestGetAndPutAuth(t *testing.T) {
fileName := filepath.Base(tempPath)
hostname, err := os.Hostname()
assert.NoError(t, err)
uploadURL := "pelican://"+ hostname + ":8444/test/" + fileName
uploadURL := "pelican://" + hostname + ":8444/test/" + fileName

// Upload the file with PUT
ObjectClientOptions.Token = tempToken.Name()
Expand All @@ -797,7 +797,7 @@ func TestGetAndPutAuth(t *testing.T) {
})

// This tests pelican object get/put with an osdf url
// NOTE: this test MUST be last since we need to reset a good amount of config values to get osdf defaults
// NOTE: this test MUST be last since we need to reset a good amount of config values to get osdf defaults as well as re-init config and client
t.Run("testOsdfObjectPutAndGetWithOSDFUrl", func(t *testing.T) {
viper.Reset()
config.SetPreferredPrefix("OSDF")
Expand All @@ -811,13 +811,13 @@ func TestGetAndPutAuth(t *testing.T) {
assert.NoError(t, err)

// For OSDF url's, we don't want to rely on osdf metadata to be running therefore, just ensure we get correct metadata for the url:
metadata, err := getUrlMetadata(uploadURL, "osdf")
pelicanURL, err := newPelicanURL(uploadURL, "osdf")
assert.NoError(t, err)

// Check valid metadata:
assert.Equal(t, "https://osdf-director.osg-htc.org", metadata.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", metadata.registryUrl)
assert.Equal(t, "osg-htc.org", metadata.discoveryUrl)
assert.Equal(t, "https://osdf-director.osg-htc.org", pelicanURL.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", pelicanURL.registryUrl)
assert.Equal(t, "osg-htc.org", pelicanURL.discoveryUrl)
viper.Reset()
})
}
Expand All @@ -832,20 +832,20 @@ func TestGetPublicRead(t *testing.T) {
t.Run("testPubObjGet", func(t *testing.T) {
testFileContent := "test file content"
// Drop the testFileContent into the origin directory
tempFile, err := os.Create(filepath.Join(fed.OriginDir, "test.txt"))
tempFile, err := os.Create(filepath.Join(fed.OriginDir, "test1234.txt"))
assert.NoError(t, err, "Error creating temp file")
defer os.Remove(tempFile.Name())
_, err = tempFile.WriteString(testFileContent)
assert.NoError(t, err, "Error writing to temp file")
tempFile.Close()

viper.Set("Logging.DisableProgressBars", true)
ObjectClientOptions.ProgressBars = false

// Set path for object to upload/download
tempPath := tempFile.Name()
hostname, err := os.Hostname()
fileName := filepath.Base(tempPath)
uploadURL := "pelican://"+ hostname + ":8444/test/" + fileName
uploadURL := "pelican://" + hostname + ":8444/test/" + fileName

// Download the file with GET. Shouldn't need a token to succeed
transferResults, err := DoGet(uploadURL, t.TempDir(), false)
Expand Down Expand Up @@ -894,7 +894,7 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
tempToken.Close()

// Disable progress bars to not reuse the same mpb instance
viper.Set("Logging.DisableProgressBars", true)
ObjectClientOptions.ProgressBars = false

// Make our test directories and files
tempDir, err := os.MkdirTemp("", "UploadDir")
Expand Down Expand Up @@ -929,13 +929,13 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
assert.NoError(t, err)

// For OSDF url's, we don't want to rely on osdf metadata to be running therefore, just ensure we get correct metadata for the url:
metadata, err := getUrlMetadata(uploadURL, "osdf")
pelicanURL, err := newPelicanURL(uploadURL, "osdf")
assert.NoError(t, err)

// Check valid metadata:
assert.Equal(t, "https://osdf-director.osg-htc.org", metadata.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", metadata.registryUrl)
assert.Equal(t, "osg-htc.org", metadata.discoveryUrl)
assert.Equal(t, "https://osdf-director.osg-htc.org", pelicanURL.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", pelicanURL.registryUrl)
assert.Equal(t, "osg-htc.org", pelicanURL.discoveryUrl)
})

t.Run("testPelicanRecursiveGetAndPutPelicanURL", func(t *testing.T) {
Expand All @@ -946,7 +946,7 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
dirName := filepath.Base(tempPath)
hostname, err := os.Hostname()
assert.NoError(t, err)
uploadURL := "pelican://"+ hostname + ":8444/test/" + dirName
uploadURL := "pelican://" + hostname + ":8444/test/" + dirName

//////////////////////////////////////////////////////////

Expand Down Expand Up @@ -1019,7 +1019,7 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
dirName := filepath.Base(tempPath)
hostname, err := os.Hostname()
assert.NoError(t, err)
uploadURL := "pelican://"+ hostname + ":8444/test/" + dirName
uploadURL := "pelican://" + hostname + ":8444/test/" + dirName

//////////////////////////////////////////////////////////

Expand Down Expand Up @@ -1098,12 +1098,12 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
assert.NoError(t, err)

// For OSDF url's, we don't want to rely on osdf metadata to be running therefore, just ensure we get correct metadata for the url:
metadata, err := getUrlMetadata(uploadURL, "osdf")
pelicanURL, err := newPelicanURL(uploadURL, "osdf")
assert.NoError(t, err)

// Check valid metadata:
assert.Equal(t, "https://osdf-director.osg-htc.org", metadata.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", metadata.registryUrl)
assert.Equal(t, "osg-htc.org", metadata.discoveryUrl)
assert.Equal(t, "https://osdf-director.osg-htc.org", pelicanURL.directorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", pelicanURL.registryUrl)
assert.Equal(t, "osg-htc.org", pelicanURL.discoveryUrl)
})
}
Loading

0 comments on commit 144ec34

Please sign in to comment.