diff --git a/flytepropeller/pkg/controller/nodes/catalog/datacatalog/datacatalog.go b/flytepropeller/pkg/controller/nodes/catalog/datacatalog/datacatalog.go index 546aeff8a8c..de520d2b7a9 100644 --- a/flytepropeller/pkg/controller/nodes/catalog/datacatalog/datacatalog.go +++ b/flytepropeller/pkg/controller/nodes/catalog/datacatalog/datacatalog.go @@ -230,7 +230,7 @@ func (m *CatalogClient) createArtifact(ctx context.Context, key catalog.Key, dat createArtifactRequest := &datacatalog.CreateArtifactRequest{Artifact: cachedArtifact} _, err := m.client.CreateArtifact(ctx, createArtifactRequest) if err != nil { - logger.Errorf(ctx, "Failed to create Artifact %+v, err: %v", cachedArtifact, err) + logger.Errorf(ctx, "Failed to create Artifact %+v, err: %v", cachedArtifact.Id, err) return catalog.Status{}, err } logger.Debugf(ctx, "Created artifact: %v, with %v outputs from execution %+v", cachedArtifact.Id, len(artifactDataList), metadata) diff --git a/flytepropeller/pkg/controller/nodes/common/utils.go b/flytepropeller/pkg/controller/nodes/common/utils.go index e5842ab1a9a..ae5145ef3bb 100644 --- a/flytepropeller/pkg/controller/nodes/common/utils.go +++ b/flytepropeller/pkg/controller/nodes/common/utils.go @@ -2,7 +2,9 @@ package common import ( "context" + "encoding/base64" "fmt" + "strconv" "github.com/golang/protobuf/proto" @@ -17,6 +19,7 @@ import ( "github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/handler" "github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/interfaces" "github.com/flyteorg/flyte/flytestdlib/logger" + "github.com/flyteorg/flyte/flytestdlib/pbhash" "github.com/flyteorg/flyte/flytestdlib/storage" ) @@ -125,6 +128,12 @@ func OffloadLargeLiteral(ctx context.Context, datastore *storage.DataStore, data logger.Errorf(ctx, "Failed to offload literal at location [%s] with error [%s]", dataReference, err) return err } + // compute the hash of the literal + literalDigest, err := pbhash.ComputeHash(ctx, toBeOffloaded) + if err != nil { + logger.Errorf(ctx, "Failed to compute hash for offloaded literal with error [%s]", err) + return err + } // update the literal with the offloaded URI, size and inferred type toBeOffloaded.Value = &idlcore.Literal_OffloadedMetadata{ @@ -134,6 +143,10 @@ func OffloadLargeLiteral(ctx context.Context, datastore *storage.DataStore, data InferredType: inferredType, }, } + if toBeOffloaded.GetHash() == "" { + // Set the hash or else respect what the user set in the literal + toBeOffloaded.Hash = base64.RawURLEncoding.EncodeToString(literalDigest) + } logger.Infof(ctx, "Offloaded literal at location [%s] with size [%d] MB and inferred type [%s]", dataReference, literalSizeMB, inferredType) return nil }