From 7056164c446c428f33d7d6c8cbfd7ed641b54583 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 25 Apr 2024 15:32:55 -0400 Subject: [PATCH] feat: set service v0 (#2372) ## Description Adds `set_service` instruction which can be used to override the service config of a service added earlier in the plan. This can be really useful for two cases: 1. A user wants to set a specific image (or other config) for a service in a package but the package author has not made the image configurable. This can be done by appending `set_service` to the plan. ex. ``` postgres = import_module("github.com/kurtosis-tech/postgres-package/main.star") def run(plan, args): postgres.run(plan) plan.set_service(name="postgres", config=ServiceConfig(image="postgres:bullseye")) ``` 2. A user wants to swap out the image (or other config) of a service running in an enclave. This can be done by appending `set_service` to the plan and enclave editing. ex. https://www.loom.com/share/f88d0f5555dd49b0aeabc55c5ed41d43 This instruction has some functionality missing such as overriding ports, env vars, and cmd/entrypoint as these require special handling of future references and handling consecutive set service instructions properly but functionality can be added iteratively in future PRs. ## Is this change user facing? YES ## References: https://github.com/kurtosis-tech/kurtosis/issues/2057 --- .../objects/service/service_config.go | 63 ++++- .../interpretation_time_value_store.go | 41 ++- .../interpretation_time_value_store_test.go | 107 ++++++++ .../startosis_engine/kurtosis_builtins.go | 2 + .../add_service/add_service.go | 51 ++-- .../get_services/get_services.go | 1 + .../set_service/set_service.go | 249 ++++++++++++++++++ .../shared_helpers/service_helpers.go | 38 +++ .../kurtosis_plan_instruction.go | 2 +- .../test_engine/set_service_framework_test.go | 89 +++++++ .../startosis_engine/startosis_interpreter.go | 3 +- .../startosis_interpreter_idempotent_test.go | 148 +++++++++++ .../startosis_interpreter_test.go | 44 ++++ .../api-reference/starlark-reference/plan.md | 49 +++- 14 files changed, 841 insertions(+), 46 deletions(-) create mode 100644 core/server/api_container/server/startosis_engine/interpretation_time_value_store/interpretation_time_value_store_test.go create mode 100644 core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go create mode 100644 core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/set_service_framework_test.go diff --git a/container-engine-lib/lib/backend_interface/objects/service/service_config.go b/container-engine-lib/lib/backend_interface/objects/service/service_config.go index f3cb33d100..94560c1f22 100644 --- a/container-engine-lib/lib/backend_interface/objects/service/service_config.go +++ b/container-engine-lib/lib/backend_interface/objects/service/service_config.go @@ -29,9 +29,12 @@ type privateServiceConfig struct { // Configuration for container engine to pull an in a private registry behind authentication // If nil, we will use the ContainerImageName and not use any auth - // Mutually exclusive from ImageBuildSpec, ContainerImageName - ImagerRegistrySpec *image_registry_spec.ImageRegistrySpec + // Mutually exclusive from ImageBuildSpec, ContainerImageName, NixBuildSpec + ImageRegistrySpec *image_registry_spec.ImageRegistrySpec + // Configuration for container engine to using Nix + // If nil, we will use the ContainerImageName and not use any Nix + // Mutually exclusive from ImageBuildSpec, ContainerImageName, ImageRegistrySpec NixBuildSpec *nix_build_spec.NixBuildSpec PrivatePorts map[string]*port_spec.PortSpec @@ -88,7 +91,7 @@ func CreateServiceConfig( cpuAllocationMillicpus uint64, memoryAllocationMegabytes uint64, privateIPAddrPlaceholder string, - minCpuMilliCores uint64, + minCpuMilliCpus uint64, minMemoryMegaBytes uint64, labels map[string]string, user *service_user.ServiceUser, @@ -104,7 +107,7 @@ func CreateServiceConfig( internalServiceConfig := &privateServiceConfig{ ContainerImageName: containerImageName, ImageBuildSpec: imageBuildSpec, - ImagerRegistrySpec: imageRegistrySpec, + ImageRegistrySpec: imageRegistrySpec, NixBuildSpec: nixBuildSpec, PrivatePorts: privatePorts, PublicPorts: publicPorts, @@ -117,7 +120,7 @@ func CreateServiceConfig( MemoryAllocationMegabytes: memoryAllocationMegabytes, PrivateIPAddrPlaceholder: privateIPAddrPlaceholder, // The minimum resources specification is only available for kubernetes - MinCpuAllocationMilliCpus: minCpuMilliCores, + MinCpuAllocationMilliCpus: minCpuMilliCpus, MinMemoryAllocationMegabytes: minMemoryMegaBytes, Labels: labels, User: user, @@ -133,18 +136,34 @@ func (serviceConfig *ServiceConfig) GetContainerImageName() string { return serviceConfig.privateServiceConfig.ContainerImageName } +func (serviceConfig *ServiceConfig) SetContainerImageName(containerImage string) { + serviceConfig.privateServiceConfig.ContainerImageName = containerImage +} + func (serviceConfig *ServiceConfig) GetImageBuildSpec() *image_build_spec.ImageBuildSpec { return serviceConfig.privateServiceConfig.ImageBuildSpec } +func (serviceConfig *ServiceConfig) SetImageBuildSpec(imageBuildSpec *image_build_spec.ImageBuildSpec) { + serviceConfig.privateServiceConfig.ImageBuildSpec = imageBuildSpec +} + func (serviceConfig *ServiceConfig) GetImageRegistrySpec() *image_registry_spec.ImageRegistrySpec { - return serviceConfig.privateServiceConfig.ImagerRegistrySpec + return serviceConfig.privateServiceConfig.ImageRegistrySpec +} + +func (serviceConfig *ServiceConfig) SetImageRegistrySpec(imageRegistrySpec *image_registry_spec.ImageRegistrySpec) { + serviceConfig.privateServiceConfig.ImageRegistrySpec = imageRegistrySpec } func (serviceConfig *ServiceConfig) GetNixBuildSpec() *nix_build_spec.NixBuildSpec { return serviceConfig.privateServiceConfig.NixBuildSpec } +func (serviceConfig *ServiceConfig) SetNixBuildSpec(nixBuildSpec *nix_build_spec.NixBuildSpec) { + serviceConfig.privateServiceConfig.NixBuildSpec = nixBuildSpec +} + func (serviceConfig *ServiceConfig) GetPrivatePorts() map[string]*port_spec.PortSpec { return serviceConfig.privateServiceConfig.PrivatePorts } @@ -177,10 +196,18 @@ func (serviceConfig *ServiceConfig) GetCPUAllocationMillicpus() uint64 { return serviceConfig.privateServiceConfig.CpuAllocationMillicpus } +func (serviceConfig *ServiceConfig) SetCPUAllocationMillicpus(cpuAllocation uint64) { + serviceConfig.privateServiceConfig.CpuAllocationMillicpus = cpuAllocation +} + func (serviceConfig *ServiceConfig) GetMemoryAllocationMegabytes() uint64 { return serviceConfig.privateServiceConfig.MemoryAllocationMegabytes } +func (serviceConfig *ServiceConfig) SetMemoryAllocationMegabytes(memoryAllocation uint64) { + serviceConfig.privateServiceConfig.MemoryAllocationMegabytes = memoryAllocation +} + func (serviceConfig *ServiceConfig) GetPrivateIPAddrPlaceholder() string { return serviceConfig.privateServiceConfig.PrivateIPAddrPlaceholder } @@ -190,27 +217,51 @@ func (serviceConfig *ServiceConfig) GetMinCPUAllocationMillicpus() uint64 { return serviceConfig.privateServiceConfig.MinCpuAllocationMilliCpus } +func (serviceConfig *ServiceConfig) SetMinCPUAllocationMillicpus(cpuAllocation uint64) { + serviceConfig.privateServiceConfig.MinCpuAllocationMilliCpus = cpuAllocation +} + // only available for Kubernetes func (serviceConfig *ServiceConfig) GetMinMemoryAllocationMegabytes() uint64 { return serviceConfig.privateServiceConfig.MinMemoryAllocationMegabytes } +func (serviceConfig *ServiceConfig) SetMinMemoryAllocationMegabytes(memoryAllocation uint64) { + serviceConfig.privateServiceConfig.MemoryAllocationMegabytes = memoryAllocation +} + func (serviceConfig *ServiceConfig) GetUser() *service_user.ServiceUser { return serviceConfig.privateServiceConfig.User } +func (serviceConfig *ServiceConfig) SetUser(user *service_user.ServiceUser) { + serviceConfig.privateServiceConfig.User = user +} + func (serviceConfig *ServiceConfig) GetLabels() map[string]string { return serviceConfig.privateServiceConfig.Labels } +func (serviceConfig *ServiceConfig) SetLabels(labels map[string]string) { + serviceConfig.privateServiceConfig.Labels = labels +} + func (serviceConfig *ServiceConfig) GetTolerations() []v1.Toleration { return serviceConfig.privateServiceConfig.Tolerations } +func (serviceConfig *ServiceConfig) SetTolerations(tolerations []v1.Toleration) { + serviceConfig.privateServiceConfig.Tolerations = tolerations +} + func (serviceConfig *ServiceConfig) GetImageDownloadMode() image_download_mode.ImageDownloadMode { return serviceConfig.privateServiceConfig.ImageDownloadMode } +func (serviceConfig *ServiceConfig) SetImageDownloadMode(mode image_download_mode.ImageDownloadMode) { + serviceConfig.privateServiceConfig.ImageDownloadMode = mode +} + func (serviceConfig *ServiceConfig) MarshalJSON() ([]byte, error) { return json.Marshal(serviceConfig.privateServiceConfig) } diff --git a/core/server/api_container/server/startosis_engine/interpretation_time_value_store/interpretation_time_value_store.go b/core/server/api_container/server/startosis_engine/interpretation_time_value_store/interpretation_time_value_store.go index a6bb42bb28..7ea9035a14 100644 --- a/core/server/api_container/server/startosis_engine/interpretation_time_value_store/interpretation_time_value_store.go +++ b/core/server/api_container/server/startosis_engine/interpretation_time_value_store/interpretation_time_value_store.go @@ -8,8 +8,10 @@ import ( ) type InterpretationTimeValueStore struct { - serviceValues *serviceInterpretationValueRepository - serde *kurtosis_types.StarlarkValueSerde + serviceConfigValues map[service.ServiceName]*service.ServiceConfig + setServiceConfigValues map[service.ServiceName]*service.ServiceConfig + serviceValues *serviceInterpretationValueRepository + serde *kurtosis_types.StarlarkValueSerde } func CreateInterpretationTimeValueStore(enclaveDb *enclave_db.EnclaveDB, serde *kurtosis_types.StarlarkValueSerde) (*InterpretationTimeValueStore, error) { @@ -17,7 +19,11 @@ func CreateInterpretationTimeValueStore(enclaveDb *enclave_db.EnclaveDB, serde * if err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating interpretation time value store") } - return &InterpretationTimeValueStore{serviceValues: serviceValuesRepository, serde: serde}, nil + return &InterpretationTimeValueStore{ + serviceConfigValues: map[service.ServiceName]*service.ServiceConfig{}, + setServiceConfigValues: map[service.ServiceName]*service.ServiceConfig{}, + serviceValues: serviceValuesRepository, + serde: serde}, nil } func (itvs *InterpretationTimeValueStore) PutService(name service.ServiceName, service *kurtosis_types.Service) error { @@ -50,3 +56,32 @@ func (itvs *InterpretationTimeValueStore) RemoveService(name service.ServiceName } return nil } + +func (itvs *InterpretationTimeValueStore) PutServiceConfig(name service.ServiceName, serviceConfig *service.ServiceConfig) { + itvs.serviceConfigValues[name] = serviceConfig +} + +func (itvs *InterpretationTimeValueStore) GetServiceConfig(name service.ServiceName) (*service.ServiceConfig, error) { + serviceConfig, ok := itvs.serviceConfigValues[name] + if !ok { + return nil, stacktrace.NewError("Did not find new service config for '%v' in interpretation time value store.", name) + } + return serviceConfig, nil +} + +func (itvs *InterpretationTimeValueStore) SetServiceConfig(name service.ServiceName, serviceConfig *service.ServiceConfig) { + itvs.setServiceConfigValues[name] = serviceConfig +} + +func (itvs *InterpretationTimeValueStore) ExistsNewServiceConfigForService(name service.ServiceName) bool { + _, doesConfigFromSetServiceInstructionExists := itvs.setServiceConfigValues[name] + return doesConfigFromSetServiceInstructionExists +} + +func (itvs *InterpretationTimeValueStore) GetNewServiceConfig(name service.ServiceName) (*service.ServiceConfig, error) { + newServiceConfig, ok := itvs.setServiceConfigValues[name] + if !ok { + return nil, stacktrace.NewError("Did not find new service config for '%v' in interpretation time value store.", name) + } + return newServiceConfig, nil +} diff --git a/core/server/api_container/server/startosis_engine/interpretation_time_value_store/interpretation_time_value_store_test.go b/core/server/api_container/server/startosis_engine/interpretation_time_value_store/interpretation_time_value_store_test.go new file mode 100644 index 0000000000..49ef32590d --- /dev/null +++ b/core/server/api_container/server/startosis_engine/interpretation_time_value_store/interpretation_time_value_store_test.go @@ -0,0 +1,107 @@ +package interpretation_time_value_store + +import ( + "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers" + "github.com/stretchr/testify/require" + bolt "go.etcd.io/bbolt" + "os" + "testing" +) + +const ( + testServiceName = service.ServiceName("datastore-service") + testContainerImageName = "datastore-image" + enclaveDbFilePerm = 0666 +) + +func TestGetServiceConfigReturnsError(t *testing.T) { + enclaveDb := getEnclaveDBForTest(t) + dummySerde := shared_helpers.NewDummyStarlarkValueSerDeForTest() + itvs, err := CreateInterpretationTimeValueStore(enclaveDb, dummySerde) + require.NoError(t, err) + + // no service config exists in store + _, err = itvs.GetServiceConfig(testServiceName) + require.Error(t, err) +} + +func TestPutServiceConfig(t *testing.T) { + enclaveDb := getEnclaveDBForTest(t) + dummySerde := shared_helpers.NewDummyStarlarkValueSerDeForTest() + itvs, err := CreateInterpretationTimeValueStore(enclaveDb, dummySerde) + require.NoError(t, err) + + expectedServiceConfig, err := getTestServiceConfigForService(testServiceName, "latest") + require.NoError(t, err) + + itvs.PutServiceConfig(testServiceName, expectedServiceConfig) + + actualServiceConfig, err := itvs.GetServiceConfig(testServiceName) + require.NoError(t, err) + require.Equal(t, expectedServiceConfig.GetContainerImageName(), actualServiceConfig.GetContainerImageName()) +} + +func TestPutNewServiceConfig(t *testing.T) { + enclaveDb := getEnclaveDBForTest(t) + dummySerde := shared_helpers.NewDummyStarlarkValueSerDeForTest() + itvs, err := CreateInterpretationTimeValueStore(enclaveDb, dummySerde) + require.NoError(t, err) + + oldServiceConfig, err := getTestServiceConfigForService(testServiceName, "older") + require.NoError(t, err) + itvs.PutServiceConfig(testServiceName, oldServiceConfig) + + newerServiceConfig, err := getTestServiceConfigForService(testServiceName, "latest") + require.NoError(t, err) + itvs.SetServiceConfig(testServiceName, newerServiceConfig) + + actualNewerServiceConfig, err := itvs.GetNewServiceConfig(testServiceName) + require.NoError(t, err) + require.Equal(t, newerServiceConfig.GetContainerImageName(), actualNewerServiceConfig.GetContainerImageName()) +} + +func getTestServiceConfigForService(name service.ServiceName, imageTag string) (*service.ServiceConfig, error) { + return service.CreateServiceConfig( + fmt.Sprintf("%v-%v:%v", name, testContainerImageName, imageTag), + nil, + nil, + nil, + nil, + nil, + []string{}, + []string{}, + map[string]string{}, + nil, + nil, + 0, + 0, + "IP-ADDRESS", + 0, + 0, + map[string]string{}, + nil, + nil, + nil, + image_download_mode.ImageDownloadMode_Always) +} + +func getEnclaveDBForTest(t *testing.T) *enclave_db.EnclaveDB { + file, err := os.CreateTemp("/tmp", "*.db") + defer func() { + err = os.Remove(file.Name()) + require.NoError(t, err) + }() + + require.NoError(t, err) + db, err := bolt.Open(file.Name(), enclaveDbFilePerm, nil) + require.NoError(t, err) + enclaveDb := &enclave_db.EnclaveDB{ + DB: db, + } + + return enclaveDb +} diff --git a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go index e86ee10df3..2fec5f5555 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go @@ -17,6 +17,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/request" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/start_service" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/stop_service" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files" @@ -73,6 +74,7 @@ func KurtosisPlanInstructions( add_service.NewAddServices(serviceNetwork, runtimeValueStore, packageId, packageContentProvider, packageReplaceOptions, interpretationTimeValueStore, imageDownloadMode), get_service.NewGetService(interpretationTimeValueStore), get_services.NewGetServices(interpretationTimeValueStore), + set_service.NewSetService(serviceNetwork, interpretationTimeValueStore, packageId, packageContentProvider, packageReplaceOptions, imageDownloadMode), get_files_artifact.NewGetFilesArtifact(), verify.NewVerify(runtimeValueStore), exec.NewExec(serviceNetwork, runtimeValueStore), diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go index ec34e25a92..903b30c358 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go @@ -9,6 +9,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_plan_persistence" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_structure" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/interpretation_time_value_store" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction" @@ -140,7 +141,7 @@ func (builtin *AddServiceCapabilities) Interpret(locatorOfModuleInWhichThisBuilt return nil, startosis_errors.NewInterpretationError("Unable to extract image attribute off of service config.") } builtin.imageVal = rawImageVal - apiServiceConfig, readyCondition, interpretationErr := validateAndConvertConfigAndReadyCondition( + apiServiceConfig, readyCondition, interpretationErr := shared_helpers.ValidateAndConvertConfigAndReadyCondition( builtin.serviceNetwork, serviceConfig, locatorOfModuleInWhichThisBuiltInIsBeingCalled, @@ -172,6 +173,8 @@ func (builtin *AddServiceCapabilities) Interpret(locatorOfModuleInWhichThisBuilt if err != nil { return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred while persisting return value for service '%v'", serviceName) } + builtin.interpretationTimeValueStore.PutServiceConfig(builtin.serviceName, builtin.serviceConfig) + return builtin.returnValue, nil } @@ -183,6 +186,14 @@ func (builtin *AddServiceCapabilities) Validate(_ *builtin_argument.ArgumentValu } func (builtin *AddServiceCapabilities) Execute(ctx context.Context, _ *builtin_argument.ArgumentValuesSet) (string, error) { + // update service config to use new service config set by a set_service instruction, if one exists + if builtin.interpretationTimeValueStore.ExistsNewServiceConfigForService(builtin.serviceName) { + newServiceConfig, err := builtin.interpretationTimeValueStore.GetNewServiceConfig(builtin.serviceName) + if err != nil { + return "", stacktrace.Propagate(err, "An error occurred retrieving a new service config '%s'.", builtin.serviceName) + } + builtin.serviceConfig = newServiceConfig + } replacedServiceName, replacedServiceConfig, err := replaceMagicStrings(builtin.runtimeValueStore, builtin.serviceName, builtin.serviceConfig) if err != nil { return "", stacktrace.Propagate(err, "An error occurred replace a magic string in '%s' instruction arguments for service '%s'. Execution cannot proceed", AddServiceBuiltinName, builtin.serviceName) @@ -257,6 +268,12 @@ func (builtin *AddServiceCapabilities) TryResolveWith(instructionsAreEqual bool, } } + // We check if service config was changed by a set_service instruction. If that's the case, it should be rerun + if builtin.interpretationTimeValueStore.ExistsNewServiceConfigForService(builtin.serviceName) { + enclaveComponents.AddService(builtin.serviceName, enclave_structure.ComponentIsUpdated) + return enclave_structure.InstructionIsUpdate + } + enclaveComponents.AddService(builtin.serviceName, enclave_structure.ComponentWasLeftIntact) return enclave_structure.InstructionIsEqual } @@ -307,35 +324,3 @@ func (builtin *AddServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) func (builtin *AddServiceCapabilities) Description() string { return builtin.description } - -func validateAndConvertConfigAndReadyCondition( - serviceNetwork service_network.ServiceNetwork, - rawConfig starlark.Value, - locatorOfModuleInWhichThisBuiltInIsBeingCalled string, - packageId string, - packageContentProvider startosis_packages.PackageContentProvider, - packageReplaceOptions map[string]string, - imageDownloadMode image_download_mode.ImageDownloadMode, -) (*service.ServiceConfig, *service_config.ReadyCondition, *startosis_errors.InterpretationError) { - config, ok := rawConfig.(*service_config.ServiceConfig) - if !ok { - return nil, nil, startosis_errors.NewInterpretationError("The '%s' argument is not a ServiceConfig (was '%s').", ConfigsArgName, reflect.TypeOf(rawConfig)) - } - apiServiceConfig, interpretationErr := config.ToKurtosisType( - serviceNetwork, - locatorOfModuleInWhichThisBuiltInIsBeingCalled, - packageId, - packageContentProvider, - packageReplaceOptions, - imageDownloadMode) - if interpretationErr != nil { - return nil, nil, interpretationErr - } - - readyCondition, interpretationErr := config.GetReadyCondition() - if interpretationErr != nil { - return nil, nil, interpretationErr - } - - return apiServiceConfig, readyCondition, nil -} diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go index 403f7e1771..05d2667a24 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go @@ -84,6 +84,7 @@ func (builtin *GetServicesCapabilities) TryResolveWith(instructionsAreEqual bool if instructionsAreEqual { return enclave_structure.InstructionIsEqual } + return enclave_structure.InstructionIsUnknown } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go new file mode 100644 index 0000000000..340efc7c9e --- /dev/null +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go @@ -0,0 +1,249 @@ +package set_service + +import ( + "context" + "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_plan_persistence" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_structure" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/interpretation_time_value_store" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_type_constructor" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator" + "go.starlark.net/starlark" + "reflect" +) + +const ( + SetServiceBuiltinName = "set_service" + ServiceNameArgName = "name" + SetServiceConfigArgName = "config" + + descriptionFormatStr = "Setting config of service '%v'" +) + +func NewSetService( + serviceNetwork service_network.ServiceNetwork, + interpretationTimeStore *interpretation_time_value_store.InterpretationTimeValueStore, + packageId string, + packageContentProvider startosis_packages.PackageContentProvider, + packageReplaceOptions map[string]string, + imageDownloadMode image_download_mode.ImageDownloadMode, +) *kurtosis_plan_instruction.KurtosisPlanInstruction { + return &kurtosis_plan_instruction.KurtosisPlanInstruction{ + KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{ + Name: SetServiceBuiltinName, + Arguments: []*builtin_argument.BuiltinArgument{ + { + Name: ServiceNameArgName, + IsOptional: false, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], + Validator: func(value starlark.Value) *startosis_errors.InterpretationError { + return builtin_argument.NonEmptyString(value, ServiceNameArgName) + }, + }, + { + Name: SetServiceConfigArgName, + IsOptional: false, + ZeroValueProvider: builtin_argument.ZeroValueProvider[*service_config.ServiceConfig], + Validator: func(value starlark.Value) *startosis_errors.InterpretationError { + // we just try to convert the configs here to validate their shape, to avoid code duplication with Interpret + _, ok := value.(*service_config.ServiceConfig) + if !ok { + return startosis_errors.NewInterpretationError("The '%s' argument is not a ServiceConfig (was '%s').", SetServiceConfigArgName, reflect.TypeOf(value)) + } + return nil + }, + }, + }, + Deprecation: nil, + }, + Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities { + return &SetServiceCapabilities{ + interpretationTimeStore: interpretationTimeStore, + serviceNetwork: serviceNetwork, + serviceName: "", // populated at interpretation time + serviceConfig: nil, // populated at interpretation time + imageVal: nil, // populated at interpretation time + packageId: packageId, + packageContentProvider: packageContentProvider, + packageReplaceOptions: packageReplaceOptions, + description: "", // populated at interpretation time + imageDownloadMode: imageDownloadMode, + } + }, + DefaultDisplayArguments: map[string]bool{ + ServiceNameArgName: true, + }, + } +} + +type SetServiceCapabilities struct { + interpretationTimeStore *interpretation_time_value_store.InterpretationTimeValueStore + serviceName service.ServiceName + serviceConfig *service.ServiceConfig + + serviceNetwork service_network.ServiceNetwork + + // These params are needed to successfully convert service config if an ImageBuildSpec was provided + packageId string + packageContentProvider startosis_packages.PackageContentProvider + packageReplaceOptions map[string]string + imageVal starlark.Value + + imageDownloadMode image_download_mode.ImageDownloadMode + description string +} + +func (builtin *SetServiceCapabilities) Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { + serviceNameArgumentValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName) + if err != nil { + return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName) + } + serviceName := service.ServiceName(serviceNameArgumentValue.GoString()) + + builtin.serviceName = serviceName + + serviceConfigOverride, err := builtin_argument.ExtractArgumentValue[*service_config.ServiceConfig](arguments, SetServiceConfigArgName) + if err != nil { + return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", SetServiceConfigArgName) + } + rawImageVal, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.Value](serviceConfigOverride.KurtosisValueTypeDefault, service_config.ImageAttr) + if interpretationErr != nil { + return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.") + } + if !found { + return nil, startosis_errors.NewInterpretationError("Unable to extract image attribute off of service config.") + } + builtin.imageVal = rawImageVal + apiServiceConfigOverride, _, interpretationErr := shared_helpers.ValidateAndConvertConfigAndReadyCondition( + builtin.serviceNetwork, + serviceConfigOverride, + locatorOfModuleInWhichThisBuiltInIsBeingCalled, + builtin.packageId, + builtin.packageContentProvider, + builtin.packageReplaceOptions, + builtin.imageDownloadMode, + ) + if interpretationErr != nil { + return nil, interpretationErr + } + + // get original service config for service and merge with apiServiceConfigOverride + currApiServiceConfig, err := builtin.interpretationTimeStore.GetServiceConfig(builtin.serviceName) + if err != nil { + return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred retrieving service config for service: %v'", builtin.serviceName) + } + + mergedServiceConfig, err := upsertServiceConfigs(currApiServiceConfig, apiServiceConfigOverride) + if err != nil { + return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred while overriding service configs in set service for service: %v", builtin.serviceName) + } + builtin.serviceConfig = mergedServiceConfig + builtin.interpretationTimeStore.SetServiceConfig(serviceName, mergedServiceConfig) + + builtin.description = builtin_argument.GetDescriptionOrFallBack(arguments, fmt.Sprintf(descriptionFormatStr, builtin.serviceName)) + return starlark.None, nil +} + +func (builtin *SetServiceCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet, validatorEnvironment *startosis_validator.ValidatorEnvironment) *startosis_errors.ValidationError { + if exists := validatorEnvironment.DoesServiceNameExist(builtin.serviceName); exists == startosis_validator.ComponentNotFound { + return startosis_errors.NewValidationError("Service '%v' required by '%v' instruction doesn't exist", builtin.serviceName, SetServiceBuiltinName) + } + return nil +} + +func (builtin *SetServiceCapabilities) Execute(_ context.Context, _ *builtin_argument.ArgumentValuesSet) (string, error) { + // Note this is a no-op. + return fmt.Sprintf("Set service config on service '%v'.", builtin.serviceName), nil +} + +func (builtin *SetServiceCapabilities) TryResolveWith(instructionsAreEqual bool, _ *enclave_plan_persistence.EnclavePlanInstruction, enclaveComponents *enclave_structure.EnclaveComponents) enclave_structure.InstructionResolutionStatus { + if instructionsAreEqual && enclaveComponents.HasServiceBeenUpdated(builtin.serviceName) { + return enclave_structure.InstructionIsUpdate + } else if instructionsAreEqual { + return enclave_structure.InstructionIsEqual + } + return enclave_structure.InstructionIsUnknown +} + +func (builtin *SetServiceCapabilities) FillPersistableAttributes(builder *enclave_plan_persistence.EnclavePlanInstructionBuilder) { + builder.SetType(SetServiceBuiltinName).AddServiceName(builtin.serviceName) +} + +func (builtin *SetServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { + // update service does not affect the plan + return nil +} + +func (builtin *SetServiceCapabilities) Description() string { + return builtin.description +} + +// Takes values set in [serviceConfigOverride] and sets them on [currServiceConfig], leaving other values of [currServiceConfig] untouched +func upsertServiceConfigs(currServiceConfig, serviceConfigOverride *service.ServiceConfig) (*service.ServiceConfig, error) { + // only one of these image values will be set, the others will be nil or empty string + // as the Starlark service config gurantees that the service config objects only has one set + currServiceConfig.SetContainerImageName(serviceConfigOverride.GetContainerImageName()) + currServiceConfig.SetImageBuildSpec(serviceConfigOverride.GetImageBuildSpec()) + currServiceConfig.SetImageRegistrySpec(serviceConfigOverride.GetImageRegistrySpec()) + currServiceConfig.SetNixBuildSpec(serviceConfigOverride.GetNixBuildSpec()) + + // for other fields, only override if they are explicitly set on serviceConfigOverride + if cpuAllocationMillicpusOverride := serviceConfigOverride.GetCPUAllocationMillicpus(); cpuAllocationMillicpusOverride != 0 { + currServiceConfig.SetCPUAllocationMillicpus(cpuAllocationMillicpusOverride) + } + if memoryAllocationMegabytesOverride := serviceConfigOverride.GetMemoryAllocationMegabytes(); memoryAllocationMegabytesOverride != 0 { + currServiceConfig.SetMemoryAllocationMegabytes(memoryAllocationMegabytesOverride) + } + if minCPUAllocationMillicpusOverride := serviceConfigOverride.GetMinCPUAllocationMillicpus(); minCPUAllocationMillicpusOverride != 0 { + currServiceConfig.SetMinCPUAllocationMillicpus(minCPUAllocationMillicpusOverride) + } + if minMemoryAllocationMegabytesOverride := serviceConfigOverride.GetMinMemoryAllocationMegabytes(); minMemoryAllocationMegabytesOverride != 0 { + currServiceConfig.SetMinMemoryAllocationMegabytes(minMemoryAllocationMegabytesOverride) + } + if userOverride := serviceConfigOverride.GetUser(); userOverride != nil { + currServiceConfig.SetUser(userOverride) + } + if labelsOverride := serviceConfigOverride.GetLabels(); len(labelsOverride) > 0 { + currServiceConfig.SetLabels(labelsOverride) + } + if tolerationsOverride := serviceConfigOverride.GetTolerations(); len(tolerationsOverride) > 0 { + currServiceConfig.SetTolerations(tolerationsOverride) + } + + // TODO: impl logic for overriding entrypoint, cmd, env vars, and ports + // TODO: note: these will require careful handling of future references that could be potentially be overriden and affect behavior + if len(serviceConfigOverride.GetEnvVars()) != 0 { + return nil, startosis_errors.NewInterpretationError("Overriding environment variables is currently not supported.") + } + if len(serviceConfigOverride.GetEntrypointArgs()) != 0 { + return nil, startosis_errors.NewInterpretationError("Overriding entrypoint args is currently not supported.") + } + if len(serviceConfigOverride.GetCmdArgs()) != 0 { + return nil, startosis_errors.NewInterpretationError("Overriding cmd args is currently not supported.") + } + if len(serviceConfigOverride.GetPrivatePorts()) != 0 { + return nil, startosis_errors.NewInterpretationError("Overriding private ports is currently not supported. ") + } + if len(serviceConfigOverride.GetPublicPorts()) != 0 { + return nil, startosis_errors.NewInterpretationError("Overriding public ports is currently not supported.") + } + if serviceConfigOverride.GetFilesArtifactsExpansion() != nil { + return nil, startosis_errors.NewInterpretationError("Overriding files artifacts is currently not supported.") + } + if serviceConfigOverride.GetPersistentDirectories() != nil { + return nil, startosis_errors.NewInterpretationError("Overriding persistent directories is currently not supported.") + } + + return currServiceConfig, nil +} diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/service_helpers.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/service_helpers.go index 706a067989..1f209cc600 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/service_helpers.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/service_helpers.go @@ -2,20 +2,26 @@ package shared_helpers import ( "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/verify" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/recipe" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/kurtosis-tech/stacktrace" "go.starlark.net/starlark" + "reflect" "time" ) const ( bufferedChannelSize = 2 starlarkThreadName = "starlark-value-serde-for-test-thread" + configArgName = "config" ) func NewDummyStarlarkValueSerDeForTest() *kurtosis_types.StarlarkValueSerde { @@ -147,3 +153,35 @@ func executeServiceAssertionWithRecipeWithTicker( } } } + +func ValidateAndConvertConfigAndReadyCondition( + serviceNetwork service_network.ServiceNetwork, + rawConfig starlark.Value, + locatorOfModuleInWhichThisBuiltInIsBeingCalled string, + packageId string, + packageContentProvider startosis_packages.PackageContentProvider, + packageReplaceOptions map[string]string, + imageDownloadMode image_download_mode.ImageDownloadMode, +) (*service.ServiceConfig, *service_config.ReadyCondition, *startosis_errors.InterpretationError) { + config, ok := rawConfig.(*service_config.ServiceConfig) + if !ok { + return nil, nil, startosis_errors.NewInterpretationError("The '%s' argument is not a ServiceConfig (was '%s').", configArgName, reflect.TypeOf(rawConfig)) + } + apiServiceConfig, interpretationErr := config.ToKurtosisType( + serviceNetwork, + locatorOfModuleInWhichThisBuiltInIsBeingCalled, + packageId, + packageContentProvider, + packageReplaceOptions, + imageDownloadMode) + if interpretationErr != nil { + return nil, nil, interpretationErr + } + + readyCondition, interpretationErr := config.GetReadyCondition() + if interpretationErr != nil { + return nil, nil, interpretationErr + } + + return apiServiceConfig, readyCondition, nil +} diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction.go index 3a11db48c7..c767707d6f 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction.go @@ -110,7 +110,7 @@ func (builtin *KurtosisPlanInstructionWrapper) CreateBuiltin() func(thread *star instructionWrapper.String(), instructionWrapper.GetPositionInOriginalScript().String()) } - if enclavePlanInstructionPulledFromMaskMaybe != nil { + if enclavePlanInstructionPulledFromMaskMaybe != nil { // why is it that the mask is invalid if this is the case? and why not make this check before adding the instruction to the plan? builtin.instructionPlanMask.MarkAsInvalid() logrus.Debugf("Marking the plan as invalid as instruction '%s' differs from '%s'", instructionWrapper.String(), enclavePlanInstructionPulledFromMaskMaybe.StarlarkCode) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/set_service_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/set_service_framework_test.go new file mode 100644 index 0000000000..d820918520 --- /dev/null +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/set_service_framework_test.go @@ -0,0 +1,89 @@ +package test_engine + +import ( + "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/interpretation_time_value_store" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider" + "github.com/stretchr/testify/require" + "go.starlark.net/starlark" + "testing" +) + +type setServiceTestCase struct { + *testing.T + serviceNetwork *service_network.MockServiceNetwork + packageContentProvider *mock_package_content_provider.MockPackageContentProvider + interpretationTimeValueStore *interpretation_time_value_store.InterpretationTimeValueStore +} + +func (suite *KurtosisPlanInstructionTestSuite) TestSetService() { + dummySerde := shared_helpers.NewDummyStarlarkValueSerDeForTest() + enclaveDb := getEnclaveDBForTest(suite.T()) + interpretationTimeValueStore, err := interpretation_time_value_store.CreateInterpretationTimeValueStore(enclaveDb, dummySerde) + require.NoError(suite.T(), err) + suite.interpretationTimeValueStore = interpretationTimeValueStore + + testServiceConfig, err := service.CreateServiceConfig( + testContainerImageName, + nil, + nil, + nil, + nil, + nil, + []string{}, + []string{}, + map[string]string{}, + nil, + nil, + 0, + 0, + "IP-ADDRESS", + 0, + 0, + map[string]string{}, + nil, + nil, + nil, + image_download_mode.ImageDownloadMode_Always) + require.NoError(suite.T(), err) + suite.interpretationTimeValueStore.PutServiceConfig(testServiceName, testServiceConfig) + + suite.run(&setServiceTestCase{ + T: suite.T(), + serviceNetwork: suite.serviceNetwork, + packageContentProvider: suite.packageContentProvider, + interpretationTimeValueStore: suite.interpretationTimeValueStore, + }) +} + +func (t *setServiceTestCase) GetInstruction() *kurtosis_plan_instruction.KurtosisPlanInstruction { + return set_service.NewSetService( + t.serviceNetwork, + t.interpretationTimeValueStore, + testModulePackageId, + t.packageContentProvider, + testNoPackageReplaceOptions, + image_download_mode.ImageDownloadMode_Missing) +} + +func (t *setServiceTestCase) GetStarlarkCode() string { + serviceConfig := fmt.Sprintf("ServiceConfig(image=%q)", testContainerImageName) + return fmt.Sprintf(`%s(%s=%q, %s=%s)`, set_service.SetServiceBuiltinName, set_service.ServiceNameArgName, testServiceName, set_service.SetServiceConfigArgName, serviceConfig) +} + +func (t *setServiceTestCase) GetStarlarkCodeForAssertion() string { + return "" +} + +func (t *setServiceTestCase) Assert(interpretationResult starlark.Value, executionResult *string) { + require.Equal(t, starlark.None, interpretationResult) + + expectedExecutionResult := fmt.Sprintf("Set service config on service '%s'.", testServiceName) + require.Regexp(t, expectedExecutionResult, *executionResult) +} diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter.go b/core/server/api_container/server/startosis_engine/startosis_interpreter.go index 6b37e26add..8660df5ce4 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter.go @@ -136,7 +136,6 @@ func (interpreter *StartosisInterpreter) InterpretAndOptimizePlan( // - if it's not successful, then the mask is not compatible with the package. Go back to step 1 var firstPossibleIndexForMatchingInstruction int if currentEnclavePlan.Size() > naiveInstructionsPlan.Size() { - firstPossibleIndexForMatchingInstruction = currentEnclavePlan.Size() - naiveInstructionsPlan.Size() } for { @@ -149,7 +148,7 @@ func (interpreter *StartosisInterpreter) InterpretAndOptimizePlan( if matchingInstructionIdx >= 0 { logrus.Debugf("Found an instruction in enclave state at index %d which matches the first instruction of the new instructions plan", matchingInstructionIdx) // we found a match - // -> First recopy store that index into the plan so that all instructions prior to this match will be + // -> First store that index into the plan so that all instructions prior to this match will be // kept in the enclave plan logrus.Debugf("Stored index of matching instructions: %d into the new plan. The instructions prior to this index in the enclave plan won't be executed but need to be kept in the enclave plan", matchingInstructionIdx) optimizedPlan.SetIndexOfFirstInstruction(matchingInstructionIdx) diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter_idempotent_test.go b/core/server/api_container/server/startosis_engine/startosis_interpreter_idempotent_test.go index 7aebb3e0cc..96ddcb8680 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter_idempotent_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter_idempotent_test.go @@ -479,6 +479,154 @@ func (suite *StartosisInterpreterIdempotentTestSuite) TestInterpretAndOptimize_U require.True(suite.T(), scheduledInstruction4.IsExecuted()) // this instruction is not affected, i.e. it won't be re-run } +func (suite *StartosisInterpreterIdempotentTestSuite) TestStartosisInterpreterIdempotent_SetService() { + initialScript := ` +def run(plan): + config = ServiceConfig( + image = "datastore-image", + ) + plan.add_service(name = "example-datastore-server", config = config) +` + + // Interpretation of the initial script to generate the current enclave plan + _, currentEnclavePlan, interpretationApiErr := suite.interpreter.Interpret( + context.Background(), + startosis_constants.PackageIdPlaceholderForStandaloneScript, + useDefaultMainFunctionName, + noPackageReplaceOptions, + startosis_constants.PlaceHolderMainFileForPlaceStandAloneScript, + initialScript, + noInputParams, + defaultNonBlockingMode, + enclave_structure.NewEnclaveComponents(), + resolver.NewInstructionsPlanMask(0), + image_download_mode.ImageDownloadMode_Missing) + require.Nil(suite.T(), interpretationApiErr) + require.Equal(suite.T(), 1, currentEnclavePlan.Size()) + convertedEnclavePlan := suite.convertInstructionPlanToEnclavePlan(currentEnclavePlan) + + updatedScript := ` +def run(plan): + config = ServiceConfig( + image = "datastore-image", + ) + plan.add_service(name="example-datastore-server", config=config) + + newConfig = ServiceConfig( + image = "datastore-image:latest", + ) + plan.set_service(name="example-datastore-server", config=newConfig) +` + + // Interpret the updated script against the current enclave plan + _, instructionsPlan, interpretationError := suite.interpreter.InterpretAndOptimizePlan( + context.Background(), + startosis_constants.PackageIdPlaceholderForStandaloneScript, + noPackageReplaceOptions, + useDefaultMainFunctionName, + startosis_constants.PlaceHolderMainFileForPlaceStandAloneScript, + updatedScript, + noInputParams, + defaultNonBlockingMode, + convertedEnclavePlan, + image_download_mode.ImageDownloadMode_Missing, + ) + require.Nil(suite.T(), interpretationError) + + instructionSequence, err := instructionsPlan.GeneratePlan() + require.Nil(suite.T(), err) + require.Equal(suite.T(), 0, instructionsPlan.GetIndexOfFirstInstruction()) + require.Equal(suite.T(), 2, len(instructionSequence)) + + scheduledInstruction1 := instructionSequence[0] + require.Equal(suite.T(), `add_service(name="example-datastore-server", config=ServiceConfig(image="datastore-image"))`, scheduledInstruction1.GetInstruction().String()) + require.False(suite.T(), scheduledInstruction1.IsExecuted()) // the add service needs to be re-executed as the `set_service` changed the service config + + scheduledInstruction2 := instructionSequence[1] + require.Equal(suite.T(), `set_service(name="example-datastore-server", config=ServiceConfig(image="datastore-image:latest"))`, scheduledInstruction2.GetInstruction().String()) + require.False(suite.T(), scheduledInstruction2.IsExecuted()) // set service should also be re-executed because it hasn't been run, but its a noop - its effect is to swap out the service config during interpretation time +} + +func (suite *StartosisInterpreterIdempotentTestSuite) TestStartosisInterpreterIdempotent_MultipleSetService() { + initialScript := ` +def run(plan): + config = ServiceConfig( + image = "datastore-image", + ) + plan.add_service(name = "example-datastore-server", config = config) +` + + // Interpretation of the initial script to generate the current enclave plan + _, currentEnclavePlan, interpretationApiErr := suite.interpreter.Interpret( + context.Background(), + startosis_constants.PackageIdPlaceholderForStandaloneScript, + useDefaultMainFunctionName, + noPackageReplaceOptions, + startosis_constants.PlaceHolderMainFileForPlaceStandAloneScript, + initialScript, + noInputParams, + defaultNonBlockingMode, + enclave_structure.NewEnclaveComponents(), + resolver.NewInstructionsPlanMask(0), + image_download_mode.ImageDownloadMode_Missing) + require.Nil(suite.T(), interpretationApiErr) + require.Equal(suite.T(), 1, currentEnclavePlan.Size()) + convertedEnclavePlan := suite.convertInstructionPlanToEnclavePlan(currentEnclavePlan) + + updatedScript := ` +def run(plan): + config = ServiceConfig( + image = "datastore-image", + ) + plan.add_service(name="example-datastore-server", config=config) + + newConfig = ServiceConfig( + image = "datastore-image:latest", + ) + plan.set_service(name="example-datastore-server", config=newConfig) + + newerConfig = ServiceConfig( + image = "datastore-image:latest", + min_cpu=1 + ) + plan.set_service(name="example-datastore-server", config=newerConfig) +` + + // Interpret the updated script against the current enclave plan + _, instructionsPlan, interpretationError := suite.interpreter.InterpretAndOptimizePlan( + context.Background(), + startosis_constants.PackageIdPlaceholderForStandaloneScript, + noPackageReplaceOptions, + useDefaultMainFunctionName, + startosis_constants.PlaceHolderMainFileForPlaceStandAloneScript, + updatedScript, + noInputParams, + defaultNonBlockingMode, + convertedEnclavePlan, + image_download_mode.ImageDownloadMode_Missing, + ) + require.Nil(suite.T(), interpretationError) + + instructionSequence, err := instructionsPlan.GeneratePlan() + require.Nil(suite.T(), err) + require.Equal(suite.T(), 0, instructionsPlan.GetIndexOfFirstInstruction()) + require.Equal(suite.T(), 3, len(instructionSequence)) + + scheduledInstruction1 := instructionSequence[0] + require.Equal(suite.T(), `add_service(name="example-datastore-server", config=ServiceConfig(image="datastore-image"))`, scheduledInstruction1.GetInstruction().String()) + require.False(suite.T(), scheduledInstruction1.IsExecuted()) // the add service needs to be re-executed as the `set_service` changed the service config + + scheduledInstruction2 := instructionSequence[1] + require.Equal(suite.T(), `set_service(name="example-datastore-server", config=ServiceConfig(image="datastore-image:latest"))`, scheduledInstruction2.GetInstruction().String()) + require.False(suite.T(), scheduledInstruction2.IsExecuted()) // set service should be executed because it hasn't been run, but it's a noop - its effect is to swap out the service config during interpretation time + + scheduledInstruction3 := instructionSequence[2] + require.Equal(suite.T(), `set_service(name="example-datastore-server", config=ServiceConfig(image="datastore-image:latest", min_cpu=1))`, scheduledInstruction3.GetInstruction().String()) + require.False(suite.T(), scheduledInstruction3.IsExecuted()) // set service should be re-executed because it hasn't been run, but itss a noop - its effect is to swap out the service config during interpretation time + // TODO: ideally, we'd test that updated config for add_service is the most recent but we can't do that well now but we will be able to when making_set service work with ports + // TODO: in which case you can `get_service` and assert ports are from the most recent config +} + func (suite *StartosisInterpreterIdempotentTestSuite) convertInstructionPlanToEnclavePlan(instructionPlan *instructions_plan.InstructionsPlan) *enclave_plan_persistence.EnclavePlan { enclavePlan := enclave_plan_persistence.NewEnclavePlan() instructionPlanSequence, interpretationErr := instructionPlan.GeneratePlan() diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go b/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go index 7e2d1de37e..d2bea6d789 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go @@ -19,6 +19,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_print" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store" @@ -895,6 +896,49 @@ The service example-datastore-server has been removed validateScriptOutputFromPrintInstructions(suite.T(), instructionsPlan, expectedOutput) } +func (suite *StartosisInterpreterTestSuite) TestStartosisInterpreter_SetService() { + script := ` +def run(plan): + config = ServiceConfig( + image = "datastore-image", + ) + plan.add_service(name = "example-datastore-server", config = config) + + newConfig = ServiceConfig( + image = "someNewContainer", + ) + plan.set_service(name="example-datastore-server", config=newConfig) +` + + _, instructionsPlan, interpretationError := suite.interpreter.Interpret(context.Background(), startosis_constants.PackageIdPlaceholderForStandaloneScript, useDefaultMainFunctionName, noPackageReplaceOptions, startosis_constants.PlaceHolderMainFileForPlaceStandAloneScript, script, startosis_constants.EmptyInputArgs, defaultNonBlockingMode, emptyEnclaveComponents, emptyInstructionsPlanMask, defaultImageDownloadMode) + require.Nil(suite.T(), interpretationError) + require.Equal(suite.T(), 2, instructionsPlan.Size()) + + assertInstructionTypeAndPosition(suite.T(), instructionsPlan, 0, add_service.AddServiceBuiltinName, startosis_constants.PackageIdPlaceholderForStandaloneScript, 6, 18) + assertInstructionTypeAndPosition(suite.T(), instructionsPlan, 1, set_service.SetServiceBuiltinName, startosis_constants.PackageIdPlaceholderForStandaloneScript, 11, 18) +} + +func (suite *StartosisInterpreterTestSuite) TestStartosisInterpreter_SetServiceErrorsOnUnsupportedFields() { + script := ` +def run(plan): + config = ServiceConfig( + image = "datastore-image", + ) + plan.add_service(name = "example-datastore-server", config = config) + + newConfig = ServiceConfig( + image= "datastore-image", + env_vars = { + "SOME": "THING" + } + ) + plan.set_service(name="example-datastore-server", config=newConfig) +` + + _, _, interpretationError := suite.interpreter.Interpret(context.Background(), startosis_constants.PackageIdPlaceholderForStandaloneScript, useDefaultMainFunctionName, noPackageReplaceOptions, startosis_constants.PlaceHolderMainFileForPlaceStandAloneScript, script, startosis_constants.EmptyInputArgs, defaultNonBlockingMode, emptyEnclaveComponents, emptyInstructionsPlanMask, defaultImageDownloadMode) + require.NotNil(suite.T(), interpretationError) +} + func (suite *StartosisInterpreterTestSuite) TestStartosisInterpreter_NoPanicIfUploadIsPassedAPathNotOnDisk() { filePath := "github.com/kurtosis/module/lib/lib.star" script := ` diff --git a/docs/docs/api-reference/starlark-reference/plan.md b/docs/docs/api-reference/starlark-reference/plan.md index 96c1bc8a35..85ef55eb1d 100644 --- a/docs/docs/api-reference/starlark-reference/plan.md +++ b/docs/docs/api-reference/starlark-reference/plan.md @@ -128,7 +128,7 @@ service = plan.get_service( get_services ----------- -The `get_services` instruction allows you to get the [Service][service-starlark-reference] objects of running services in an enclave. This is +The `get_services` instruction allows retrieving the [Service][service-starlark-reference] objects of running services in an enclave. This is useful in situations where you are running Starlark against an existing enclave or need information about the services that an imported package started. ```python @@ -144,6 +144,53 @@ for service in services: plan.print(service.hostname) plan.print(service.ip_address) ``` + +set_service +----------- + +`set_service` instruction allows overriding the [ServiceConfig][service-config] of a service that exists in the plan. This is useful especially in cases where you are +importing a package published by somebody else, but you want to modify aspects of it. For example, if you want to change the image that a service +started by the imported package is using, but the package author has not parameterized the service image via the `run` function, you can use `set_service` to change the image! + +```python +# Returns a Service object (see the Service page in the sidebar) +plan.set_service( + # The name of the service to set the config of + # MANDATORY + name = "my-service", + + # A ServiceConfig object populated with fields to override the existing ServiceConfig object with + # OPTIONAL (Default: Fetching service 'SERVICE_NAME') + config = ServiceConfig(...), + + # A human friendly description for the end user of the package + # OPTIONAL (Default: Set service on service "my-service") + description = "Setting new image on my-service" +) +``` +Example usage: +```python +# Postgres package starts a pg db off image "postgres:latest" +postgres = import_module("github.com/kurtosis-tech/postgres-package/main.star") + +def run(plan, args): + postgres.run(plan) + + # Set service detects the "postgres" service in the plan and + # Overrides the ServiceConfig set in the `add_service` instruction of postgres package to use the new image value + plan.set_service( + name="postgres", + config=ServiceConfig( + image=ImageBuildSpec( + image_name="my-pg-db", # use my custom db instead! + build_context_dir="./database" + ) + ) + ) +``` +Note: currently, setting the ServiceConfig image, user, labels, tolerations, and resource allocation values are supported. +Overriding the ports, env vars, cmd/entrypoint args, and files are not supported. If this is something you'd like support for please let us know on [GitHub](https://github.com/kurtosis-tech/kurtosis/issues)! + get_files_artifact -----------