From 1a9d953bb26643c7b0effcf761de49bdb735a0ec Mon Sep 17 00:00:00 2001 From: Gyanendra Mishra Date: Tue, 24 Oct 2023 22:30:14 +0100 Subject: [PATCH] feat: allow for named artifact creation in run_sh and run_python (#1608) ## Description: Allows for named artifact creation via `StoreSpec` Left - [x] Clean the code - [x] Fix the docs ## Is this change user facing? YES ## References (if applicable): Closes #1581 --- .../objects/store_spec/store_spec.go | 25 +++++ .../startosis_engine/kurtosis_builtins.go | 2 + .../kurtosis_instruction/tasks/run_python.go | 44 ++++---- .../kurtosis_instruction/tasks/run_sh.go | 38 ++++--- .../tasks/tasks_shared.go | 101 +++++++++-------- .../kurtosis_types/store_spec/store_spec.go | 102 ++++++++++++++++++ docs/docs/starlark-reference/plan.md | 42 +++++--- .../run_task_sh_task_test.go | 9 +- 8 files changed, 261 insertions(+), 102 deletions(-) create mode 100644 container-engine-lib/lib/backend_interface/objects/store_spec/store_spec.go create mode 100644 core/server/api_container/server/startosis_engine/kurtosis_types/store_spec/store_spec.go diff --git a/container-engine-lib/lib/backend_interface/objects/store_spec/store_spec.go b/container-engine-lib/lib/backend_interface/objects/store_spec/store_spec.go new file mode 100644 index 0000000000..52f8e7db34 --- /dev/null +++ b/container-engine-lib/lib/backend_interface/objects/store_spec/store_spec.go @@ -0,0 +1,25 @@ +package store_spec + +type StoreSpec struct { + name string + src string +} + +func NewStoreSpec(src, name string) *StoreSpec { + return &StoreSpec{ + name: name, + src: src, + } +} + +func (storeSpec *StoreSpec) GetName() string { + return storeSpec.name +} + +func (storeSpec *StoreSpec) GetSrc() string { + return storeSpec.src +} + +func (storeSpec *StoreSpec) SetName(name string) { + storeSpec.name = name +} diff --git a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go index 571dbdae31..066d7df8ed 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go @@ -24,6 +24,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/directory" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/port_spec" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/store_spec" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/recipe" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" @@ -107,6 +108,7 @@ func KurtosisTypeConstructors() []*starlark.Builtin { starlark.NewBuiltin(recipe.GetHttpRecipeTypeName, recipe.NewGetHttpRequestRecipeType().CreateBuiltin()), starlark.NewBuiltin(recipe.PostHttpRecipeTypeName, recipe.NewPostHttpRequestRecipeType().CreateBuiltin()), starlark.NewBuiltin(port_spec.PortSpecTypeName, port_spec.NewPortSpecType().CreateBuiltin()), + starlark.NewBuiltin(store_spec.StoreSpecTypeName, store_spec.NewStoreSpecType().CreateBuiltin()), starlark.NewBuiltin(service_config.ServiceConfigTypeName, service_config.NewServiceConfigType().CreateBuiltin()), starlark.NewBuiltin(service_config.ReadyConditionTypeName, service_config.NewReadyConditionType().CreateBuiltin()), } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go index 1ca65cfe43..87d601cc59 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go @@ -7,6 +7,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_plan_persistence" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_structure" @@ -106,17 +107,16 @@ func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeV Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities { return &RunPythonCapabilities{ - serviceNetwork: serviceNetwork, - runtimeValueStore: runtimeValueStore, - pythonArguments: nil, - packages: nil, - name: "", - serviceConfig: nil, // populated at interpretation time - run: "", // populated at interpretation time - resultUuid: "", // populated at interpretation time - fileArtifactNames: nil, - pathToFileArtifacts: nil, - wait: DefaultWaitTimeoutDurationStr, + serviceNetwork: serviceNetwork, + runtimeValueStore: runtimeValueStore, + pythonArguments: nil, + packages: nil, + name: "", + serviceConfig: nil, // populated at interpretation time + run: "", // populated at interpretation time + resultUuid: "", // populated at interpretation time + storeSpecList: nil, + wait: DefaultWaitTimeoutDurationStr, } }, @@ -143,10 +143,9 @@ type RunPythonCapabilities struct { pythonArguments []string packages []string - serviceConfig *service.ServiceConfig - fileArtifactNames []string - pathToFileArtifacts []string - wait string + serviceConfig *service.ServiceConfig + storeSpecList []*store_spec.StoreSpec + wait string } func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { @@ -236,12 +235,11 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg builtin.serviceConfig = getServiceConfig(image, filesArtifactExpansion) if arguments.IsSet(StoreFilesArgName) { - pathToFileArtifacts, fileArtifactNames, interpretationErr := parseStoreFilesArg(builtin.serviceNetwork, arguments) + storeSpecList, interpretationErr := parseStoreFilesArg(builtin.serviceNetwork, arguments) if interpretationErr != nil { return nil, interpretationErr } - builtin.pathToFileArtifacts = pathToFileArtifacts - builtin.fileArtifactNames = fileArtifactNames + builtin.storeSpecList = storeSpecList } if arguments.IsSet(WaitArgName) { @@ -260,7 +258,7 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg randomUuid := uuid.NewRandom() builtin.name = fmt.Sprintf("task-%v", randomUuid.String()) - result := createInterpretationResult(resultUuid, builtin.fileArtifactNames) + result := createInterpretationResult(resultUuid, builtin.storeSpecList) return result, nil } @@ -270,7 +268,7 @@ func (builtin *RunPythonCapabilities) Validate(_ *builtin_argument.ArgumentValue if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil { serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers } - return validateTasksCommon(validatorEnvironment, builtin.fileArtifactNames, builtin.pathToFileArtifacts, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName()) + return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName()) } // Execute This is just v0 for run_python task - we can later improve on it. @@ -282,7 +280,7 @@ func (builtin *RunPythonCapabilities) Validate(_ *builtin_argument.ArgumentValue func (builtin *RunPythonCapabilities) Execute(ctx context.Context, _ *builtin_argument.ArgumentValuesSet) (string, error) { _, err := builtin.serviceNetwork.AddService(ctx, service.ServiceName(builtin.name), builtin.serviceConfig) if err != nil { - return "", stacktrace.Propagate(err, "error occurred while creating a run_sh task with image: %v", builtin.serviceConfig.GetContainerImageName()) + return "", stacktrace.Propagate(err, "error occurred while creating a run_python task with image: %v", builtin.serviceConfig.GetContainerImageName()) } pipInstallationResult, err := setupRequiredPackages(ctx, builtin) @@ -322,8 +320,8 @@ func (builtin *RunPythonCapabilities) Execute(ctx context.Context, _ *builtin_ar return "", stacktrace.NewError(formatErrorMessage(errorMessage, runPythonExecutionResult.GetOutput())) } - if builtin.fileArtifactNames != nil && builtin.pathToFileArtifacts != nil { - err = copyFilesFromTask(ctx, builtin.serviceNetwork, builtin.name, builtin.fileArtifactNames, builtin.pathToFileArtifacts) + if builtin.storeSpecList != nil { + err = copyFilesFromTask(ctx, builtin.serviceNetwork, builtin.name, builtin.storeSpecList) if err != nil { return "", stacktrace.Propagate(err, "error occurred while copying files from a task") } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go index 971686b790..5cc4050dcc 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_plan_persistence" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_structure" @@ -73,15 +74,14 @@ func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValue Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities { return &RunShCapabilities{ - serviceNetwork: serviceNetwork, - runtimeValueStore: runtimeValueStore, - name: "", - serviceConfig: nil, // populated at interpretation time - run: "", // populated at interpretation time - resultUuid: "", // populated at interpretation time - fileArtifactNames: nil, - pathToFileArtifacts: nil, - wait: DefaultWaitTimeoutDurationStr, + serviceNetwork: serviceNetwork, + runtimeValueStore: runtimeValueStore, + name: "", + serviceConfig: nil, // populated at interpretation time + run: "", // populated at interpretation time + resultUuid: "", // populated at interpretation time + storeSpecList: nil, + wait: DefaultWaitTimeoutDurationStr, } }, @@ -103,10 +103,9 @@ type RunShCapabilities struct { name string run string - serviceConfig *service.ServiceConfig - fileArtifactNames []string - pathToFileArtifacts []string - wait string + serviceConfig *service.ServiceConfig + storeSpecList []*store_spec.StoreSpec + wait string } func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { @@ -149,12 +148,11 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen builtin.serviceConfig = getServiceConfig(image, filesArtifactExpansion) if arguments.IsSet(StoreFilesArgName) { - pathToFileArtifacts, fileArtifactNames, interpretationErr := parseStoreFilesArg(builtin.serviceNetwork, arguments) + storeSpecList, interpretationErr := parseStoreFilesArg(builtin.serviceNetwork, arguments) if interpretationErr != nil { return nil, interpretationErr } - builtin.pathToFileArtifacts = pathToFileArtifacts - builtin.fileArtifactNames = fileArtifactNames + builtin.storeSpecList = storeSpecList } if arguments.IsSet(WaitArgName) { @@ -173,7 +171,7 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen randomUuid := uuid.NewRandom() builtin.name = fmt.Sprintf("task-%v", randomUuid.String()) - result := createInterpretationResult(resultUuid, builtin.fileArtifactNames) + result := createInterpretationResult(resultUuid, builtin.storeSpecList) return result, nil } @@ -183,7 +181,7 @@ func (builtin *RunShCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil { serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers } - return validateTasksCommon(validatorEnvironment, builtin.fileArtifactNames, builtin.pathToFileArtifacts, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName()) + return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName()) } // Execute This is just v0 for run_sh task - we can later improve on it. @@ -226,8 +224,8 @@ func (builtin *RunShCapabilities) Execute(ctx context.Context, _ *builtin_argume return "", stacktrace.NewError(formatErrorMessage(errorMessage, createDefaultDirectoryResult.GetOutput())) } - if builtin.fileArtifactNames != nil && builtin.pathToFileArtifacts != nil { - err = copyFilesFromTask(ctx, builtin.serviceNetwork, builtin.name, builtin.fileArtifactNames, builtin.pathToFileArtifacts) + if builtin.storeSpecList != nil { + err = copyFilesFromTask(ctx, builtin.serviceNetwork, builtin.name, builtin.storeSpecList) if err != nil { return "", stacktrace.Propagate(err, "error occurred while copying files from a task") } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go index b4ee1d7221..6b146e6a5d 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go @@ -6,11 +6,13 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/magic_string_helper" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config" + store_spec_starlark_type "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/store_spec" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator" "github.com/kurtosis-tech/stacktrace" @@ -40,40 +42,61 @@ const ( runFilesArtifactsKey = "files_artifacts" shellWrapperCommand = "/bin/sh" + noNameSet = "" + uniqueNameGenErrStr = "error occurred while generating unique name for the file artifact" ) var runTailCommandToPreventContainerToStopOnCreating = []string{"tail", "-f", "/dev/null"} -func parseStoreFilesArg(serviceNetwork service_network.ServiceNetwork, arguments *builtin_argument.ArgumentValuesSet) ([]string, []string, *startosis_errors.InterpretationError) { +func parseStoreFilesArg(serviceNetwork service_network.ServiceNetwork, arguments *builtin_argument.ArgumentValuesSet) ([]*store_spec.StoreSpec, *startosis_errors.InterpretationError) { + var result []*store_spec.StoreSpec + storeFilesList, err := builtin_argument.ExtractArgumentValue[*starlark.List](arguments, StoreFilesArgName) if err != nil { - return nil, nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", StoreFilesArgName) + return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", StoreFilesArgName) } if storeFilesList.Len() == 0 { - return nil, nil, nil + return nil, nil } - storeFilesArray, interpretationErr := kurtosis_types.SafeCastToStringSlice(storeFilesList, StoreFilesArgName) - if interpretationErr != nil { - return nil, nil, interpretationErr - } + for i := 0; i < storeFilesList.Len(); i++ { + rawStoreSpec := storeFilesList.Index(i) - pathToFileArtifacts := storeFilesArray + storeSpecObjStarlarkType, isStoreSpecObjStarlarkType := rawStoreSpec.(*store_spec_starlark_type.StoreSpec) + if isStoreSpecObjStarlarkType { + storeSpecObj, interpretationErr := storeSpecObjStarlarkType.ToKurtosisType() + if interpretationErr != nil { + return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "an error occurred while converting StoreSpec Starlark type to raw type") + } + // is a StoreSpecObj but no name was provided + if storeSpecObj.GetName() == noNameSet { + uniqueNameForArtifact, artifactCreationErr := serviceNetwork.GetUniqueNameForFileArtifact() + if artifactCreationErr != nil { + return nil, startosis_errors.WrapWithInterpretationError(artifactCreationErr, uniqueNameGenErrStr) + } + storeSpecObj.SetName(uniqueNameForArtifact) + } + result = append(result, storeSpecObj) + continue + } - // generate unique names - var uniqueNames []string - for range storeFilesArray { - uniqueNameForArtifact, err := serviceNetwork.GetUniqueNameForFileArtifact() - if err != nil { - return nil, nil, startosis_errors.WrapWithInterpretationError(err, "error occurred while generating unique name for file artifact") + // this is a pure string + storeFilesSrcStr, interpretationErr := kurtosis_types.SafeCastToString(rawStoreSpec, StoreFilesArgName) + if interpretationErr == nil { + uniqueNameForArtifact, artifactCreationErr := serviceNetwork.GetUniqueNameForFileArtifact() + if artifactCreationErr != nil { + return nil, startosis_errors.WrapWithInterpretationError(artifactCreationErr, uniqueNameGenErrStr) + } + storeSpecObj := store_spec.NewStoreSpec(storeFilesSrcStr, uniqueNameForArtifact) + result = append(result, storeSpecObj) + continue } - uniqueNames = append(uniqueNames, uniqueNameForArtifact) - } - fileArtifactNames := uniqueNames + return nil, startosis_errors.NewInterpretationError("Couldn't convert '%v' to StoreSpec type", rawStoreSpec) + } - return pathToFileArtifacts, fileArtifactNames, nil + return result, nil } func parseWaitArg(arguments *builtin_argument.ArgumentValuesSet) (string, *startosis_errors.InterpretationError) { @@ -90,7 +113,7 @@ func parseWaitArg(arguments *builtin_argument.ArgumentValuesSet) (string, *start return waitTimeout, nil } -func createInterpretationResult(resultUuid string, fileArtifactNames []string) *starlarkstruct.Struct { +func createInterpretationResult(resultUuid string, storeSpecList []*store_spec.StoreSpec) *starlarkstruct.Struct { runCodeValue := fmt.Sprintf(magic_string_helper.RuntimeValueReplacementPlaceholderFormat, resultUuid, runResultCodeKey) runOutputValue := fmt.Sprintf(magic_string_helper.RuntimeValueReplacementPlaceholderFormat, resultUuid, runResultOutputKey) @@ -100,10 +123,10 @@ func createInterpretationResult(resultUuid string, fileArtifactNames []string) * // converting go slice to starlark list artifactNamesList := &starlark.List{} - if len(fileArtifactNames) > 0 { - for _, name := range fileArtifactNames { + if len(storeSpecList) > 0 { + for _, storeSpec := range storeSpecList { // purposely not checking error for list because it's mutable so should not throw any errors until this point - _ = artifactNamesList.Append(starlark.String(name)) + _ = artifactNamesList.Append(starlark.String(storeSpec.GetName())) } } dict[runFilesArtifactsKey] = artifactNamesList @@ -111,20 +134,14 @@ func createInterpretationResult(resultUuid string, fileArtifactNames []string) * return result } -func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, fileArtifactNames []string, pathToFileArtifacts []string, serviceDirpathsToArtifactIdentifiers map[string]string, imageName string) *startosis_errors.ValidationError { - if fileArtifactNames != nil { - if len(fileArtifactNames) != len(pathToFileArtifacts) { - return startosis_errors.NewValidationError("error occurred while validating file artifact name for each file in store array. "+ - "This seems to be a bug, please create a ticket for it. names: %v paths: %v", len(fileArtifactNames), len(pathToFileArtifacts)) - } - - err := validatePathIsUniqueWhileCreatingFileArtifact(pathToFileArtifacts) - if err != nil { +func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, storeSpecList []*store_spec.StoreSpec, serviceDirpathsToArtifactIdentifiers map[string]string, imageName string) *startosis_errors.ValidationError { + if storeSpecList != nil { + if err := validatePathIsUniqueWhileCreatingFileArtifact(storeSpecList); err != nil { return startosis_errors.WrapWithValidationError(err, "error occurred while validating file paths to copy into file artifact") } - for _, name := range fileArtifactNames { - validatorEnvironment.AddArtifactName(name) + for _, storeSpec := range storeSpecList { + validatorEnvironment.AddArtifactName(storeSpec.GetName()) } } @@ -176,10 +193,11 @@ func executeWithWait(ctx context.Context, serviceNetwork service_network.Service } } -func validatePathIsUniqueWhileCreatingFileArtifact(storeFiles []string) *startosis_errors.ValidationError { - if len(storeFiles) > 0 { +func validatePathIsUniqueWhileCreatingFileArtifact(storeSpecList []*store_spec.StoreSpec) *startosis_errors.ValidationError { + if len(storeSpecList) > 0 { duplicates := map[string]uint16{} - for _, filePath := range storeFiles { + for _, storeSpec := range storeSpecList { + filePath := storeSpec.GetSrc() if duplicates[filePath] != 0 { return startosis_errors.NewValidationError( "error occurred while validating field: %v. The file paths in the array must be unique. Found multiple instances of %v", StoreFilesArgName, filePath) @@ -190,16 +208,15 @@ func validatePathIsUniqueWhileCreatingFileArtifact(storeFiles []string) *startos return nil } -func copyFilesFromTask(ctx context.Context, serviceNetwork service_network.ServiceNetwork, serviceName string, fileArtifactNames []string, pathToFileArtifacts []string) error { - if fileArtifactNames == nil || pathToFileArtifacts == nil { +func copyFilesFromTask(ctx context.Context, serviceNetwork service_network.ServiceNetwork, serviceName string, storeSpecList []*store_spec.StoreSpec) error { + if storeSpecList == nil { return nil } - for index, fileArtifactPath := range pathToFileArtifacts { - fileArtifactName := fileArtifactNames[index] - _, err := serviceNetwork.CopyFilesFromService(ctx, serviceName, fileArtifactPath, fileArtifactName) + for _, storeSpec := range storeSpecList { + _, err := serviceNetwork.CopyFilesFromService(ctx, serviceName, storeSpec.GetSrc(), storeSpec.GetName()) if err != nil { - return stacktrace.Propagate(err, fmt.Sprintf("error occurred while copying file or directory at path: %v", fileArtifactPath)) + return stacktrace.Propagate(err, fmt.Sprintf("error occurred while copying file or directory at path: %v", storeSpec.GetSrc())) } } return nil diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/store_spec/store_spec.go b/core/server/api_container/server/startosis_engine/kurtosis_types/store_spec/store_spec.go new file mode 100644 index 0000000000..f906358716 --- /dev/null +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/store_spec/store_spec.go @@ -0,0 +1,102 @@ +package store_spec + +import ( + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_type_constructor" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "go.starlark.net/starlark" +) + +const ( + StoreSpecTypeName = "StoreSpec" + + SrcAttr = "src" + NameAttr = "name" +) + +func NewStoreSpecType() *kurtosis_type_constructor.KurtosisTypeConstructor { + return &kurtosis_type_constructor.KurtosisTypeConstructor{ + KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{ + Name: StoreSpecTypeName, + Arguments: []*builtin_argument.BuiltinArgument{ + { + Name: SrcAttr, + IsOptional: false, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], + Validator: func(value starlark.Value) *startosis_errors.InterpretationError { + return builtin_argument.NonEmptyString(value, kurtosis_types.ServiceNameAttr) + }, + }, + { + Name: NameAttr, + IsOptional: true, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], + Validator: func(value starlark.Value) *startosis_errors.InterpretationError { + return builtin_argument.NonEmptyString(value, kurtosis_types.ServiceNameAttr) + }, + }, + }, + Deprecation: nil, + }, + Instantiate: instantiate, + } +} + +func instantiate(arguments *builtin_argument.ArgumentValuesSet) (builtin_argument.KurtosisValueType, *startosis_errors.InterpretationError) { + kurtosisValueType, interpretationErr := kurtosis_type_constructor.CreateKurtosisStarlarkTypeDefault(StoreSpecTypeName, arguments) + if interpretationErr != nil { + return nil, interpretationErr + } + return &StoreSpec{ + KurtosisValueTypeDefault: kurtosisValueType, + }, nil +} + +type StoreSpec struct { + *kurtosis_type_constructor.KurtosisValueTypeDefault +} + +func (storeSpecObj *StoreSpec) Copy() (builtin_argument.KurtosisValueType, error) { + copiedValueType, err := storeSpecObj.KurtosisValueTypeDefault.Copy() + if err != nil { + return nil, err + } + return &StoreSpec{ + KurtosisValueTypeDefault: copiedValueType, + }, nil +} + +func (storeSpecObj *StoreSpec) GetName() (string, *startosis_errors.InterpretationError) { + name, _, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String]( + storeSpecObj.KurtosisValueTypeDefault, NameAttr) + if interpretationErr != nil { + return "", interpretationErr + } + return name.GoString(), nil +} + +func (storeSpecObj *StoreSpec) GetSrc() (string, *startosis_errors.InterpretationError) { + src, _, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String]( + storeSpecObj.KurtosisValueTypeDefault, SrcAttr) + if interpretationErr != nil { + return "", interpretationErr + } + return src.GoString(), nil +} + +func (storeSpecObj *StoreSpec) ToKurtosisType() (*store_spec.StoreSpec, *startosis_errors.InterpretationError) { + name, interpretationErr := storeSpecObj.GetName() + if interpretationErr != nil { + return nil, interpretationErr + } + + src, interpretationErr := storeSpecObj.GetSrc() + if interpretationErr != nil { + return nil, interpretationErr + } + + return store_spec.NewStoreSpec(src, name), nil +} diff --git a/docs/docs/starlark-reference/plan.md b/docs/docs/starlark-reference/plan.md index cf3deae6e8..035e75243f 100644 --- a/docs/docs/starlark-reference/plan.md +++ b/docs/docs/starlark-reference/plan.md @@ -417,15 +417,22 @@ The `run_python` instruction executes a one-time execution task. It runs the Pyt "/path/to/file/2": files_artifact_2, }, - # list of paths to directories or files that will be copied to a file artifact - # CAUTION: all the paths in this list must be unique - # OPTIONAL (Default:[]) + # A list of filepaths to store inside files artifacts after the run_python finishes + # Entries in the list can either be a string containing the path to store, or a + # StoreSpec object that can optionally name the files artifact. + # CAUTION: Both the paths in `src` and the files artifact names must be unique! + # OPTIONAL (Default: []) store = [ - # copies a file into a file artifact - "/src/kurtosis.txt", + # EXAMPLE: Creates a files artifact named `kurtosis_txt` containing the `kurtosis.txt` file + StoreSpec(src = "/src/kurtosis.txt", name = "kurtosis_txt"), - # copies the entire directory into a file artifact - "/src", + # EXAMPLE: Creates a files artifact with an automatically-generated name containing `genesis.json` + StoreSpec(src = "/genesis.json"), + + # EXAMPLE: Creates a files artifact with an automatically-generated name containing `address.json` + # This is just syntactic sugar for: + # StoreSpec(src = "/coinbase/address.json") + "/coinbase/address.json" ], # The time to allow for the command to complete. If the Python script takes longer than this, @@ -478,15 +485,22 @@ The `run_sh` instruction executes a one-time execution task. It runs the bash co "/path/to/file/2": files_artifact_2, }, - # list of paths to directories or files that will be copied to a file artifact - # CAUTION: all the paths in this list must be unique - # OPTIONAL (Default:[]) + # A list of filepaths to store inside files artifacts after the run_sh finishes + # Entries in the list can either be a string containing the path to store, or a + # StoreSpec object that can optionally name the files artifact. + # CAUTION: Both the paths in `src` and the files artifact names must be unique! + # OPTIONAL (Default: []) store = [ - # copies a file into a file artifact - "/src/kurtosis.txt", + # EXAMPLE: Creates a files artifact named `kurtosis_txt` containing the `kurtosis.txt` file + StoreSpec(src = "/src/kurtosis.txt", name = "kurtosis_txt"), - # copies the entire directory into a file artifact - "/src", + # EXAMPLE: Creates a files artifact with an automatically-generated name containing `genesis.json` + StoreSpec(src = "/genesis.json"), + + # EXAMPLE: Creates a files artifact with an automatically-generated name containing `address.json` + # This is just syntactic sugar for: + # StoreSpec(src = "/coinbase/address.json") + "/coinbase/address.json" ], # The time to allow for the command to complete. If the command takes longer than this, diff --git a/internal_testsuites/golang/testsuite/startosis_run_sh_task_test/run_task_sh_task_test.go b/internal_testsuites/golang/testsuite/startosis_run_sh_task_test/run_task_sh_task_test.go index 1a33a9ee08..4492179510 100644 --- a/internal_testsuites/golang/testsuite/startosis_run_sh_task_test/run_task_sh_task_test.go +++ b/internal_testsuites/golang/testsuite/startosis_run_sh_task_test/run_task_sh_task_test.go @@ -17,12 +17,14 @@ def run(plan): ` runshStarlarkFileArtifact = ` def run(plan): - result = plan.run_sh(run="mkdir -p /src && echo kurtosis > /src/tech.txt", store=["/src/tech.txt", "/src"], image="ethpandaops/ethereum-genesis-generator:1.0.14") + result = plan.run_sh(run="mkdir -p /src && echo kurtosis > /src/tech.txt && echo example > /src/example.txt", store=["/src/tech.txt", StoreSpec(src="/src", name="src"), StoreSpec(src="/src/example.txt")], image="ethpandaops/ethereum-genesis-generator:1.0.14") file_artifacts = result.files_artifacts result2 = plan.run_sh(run="cat /temp/tech.txt", files={"/temp": file_artifacts[0]}) plan.verify(result2.output, "==", "kurtosis\n") result3 = plan.run_sh(run="cat /task/src/tech.txt", files={"/task": file_artifacts[1]}) plan.verify(result3.output, "==", "kurtosis\n") + result4 = plan.run_sh(run = "cat /task/example.txt", files={"/task": file_artifacts[2]}) + plan.verify(result4.output, "==", "example\n") ` runshStarlarkFileArtifactFailure = ` @@ -55,8 +57,9 @@ func TestStarlark_RunshTaskSimple(t *testing.T) { func TestStarlark_RunshTaskFileArtifact(t *testing.T) { ctx := context.Background() - runResult, _ := test_helpers.SetupSimpleEnclaveAndRunScript(t, ctx, runshTest, runshStarlarkFileArtifact) - expectedOutput := "Command returned with exit code '0' with no output\nCommand returned with exit code '0' and the following output:\n--------------------\nkurtosis\n\n--------------------\nVerification succeeded. Value is '\"kurtosis\\n\"'.\nCommand returned with exit code '0' and the following output:\n--------------------\nkurtosis\n\n--------------------\nVerification succeeded. Value is '\"kurtosis\\n\"'.\n" + runResult, err := test_helpers.SetupSimpleEnclaveAndRunScript(t, ctx, runshTest, runshStarlarkFileArtifact) + require.Nil(t, err) + expectedOutput := "Command returned with exit code '0' with no output\nCommand returned with exit code '0' and the following output:\n--------------------\nkurtosis\n\n--------------------\nVerification succeeded. Value is '\"kurtosis\\n\"'.\nCommand returned with exit code '0' and the following output:\n--------------------\nkurtosis\n\n--------------------\nVerification succeeded. Value is '\"kurtosis\\n\"'.\nCommand returned with exit code '0' and the following output:\n--------------------\nexample\n\n--------------------\nVerification succeeded. Value is '\"example\\n\"'.\n" require.Equal(t, expectedOutput, string(runResult.RunOutput)) }