diff --git a/docs/data-sources/deployment.md b/docs/data-sources/deployment.md index 366fe7f4..c7bf056b 100644 --- a/docs/data-sources/deployment.md +++ b/docs/data-sources/deployment.md @@ -91,6 +91,7 @@ Read-Only: - `credentials` (String) Credentials to use for the pull step. Refer to a {GitHub,GitLab,BitBucket} credentials block. - `directory` (String) (For type 'set_working_directory') The directory to set as the working directory. - `folder` (String) (For type 'pull_from_*') The folder in the bucket where files are stored. +- `include_submodules` (Boolean) (For type 'git_clone') Whether to include submodules when cloning the repository. - `repository` (String) (For type 'git_clone') The URL of the repository to clone. - `requires` (String) A list of Python package dependencies. - `type` (String) The type of pull step diff --git a/docs/resources/deployment.md b/docs/resources/deployment.md index e95e8ce1..764fb076 100644 --- a/docs/resources/deployment.md +++ b/docs/resources/deployment.md @@ -67,10 +67,11 @@ resource "prefect_deployment" "deployment" { directory = "/some/directory", }, { - type = "git_clone" - repository = "https://github.com/some/repo" - branch = "main" - access_token = "123abc" + type = "git_clone" + repository = "https://github.com/some/repo" + branch = "main" + access_token = "123abc" + include_submodules = true }, { type = "pull_from_s3", @@ -145,6 +146,7 @@ Optional: - `credentials` (String) Credentials to use for the pull step. Refer to a {GitHub,GitLab,BitBucket} credentials block. - `directory` (String) (For type 'set_working_directory') The directory to set as the working directory. - `folder` (String) (For type 'pull_from_*') The folder in the bucket where files are stored. +- `include_submodules` (Boolean) (For type 'git_clone') Whether to include submodules when cloning the repository. - `repository` (String) (For type 'git_clone') The URL of the repository to clone. - `requires` (String) A list of Python package dependencies. diff --git a/examples/resources/prefect_deployment/resource.tf b/examples/resources/prefect_deployment/resource.tf index 9716f1fa..8c90b915 100644 --- a/examples/resources/prefect_deployment/resource.tf +++ b/examples/resources/prefect_deployment/resource.tf @@ -52,10 +52,11 @@ resource "prefect_deployment" "deployment" { directory = "/some/directory", }, { - type = "git_clone" - repository = "https://github.com/some/repo" - branch = "main" - access_token = "123abc" + type = "git_clone" + repository = "https://github.com/some/repo" + branch = "main" + access_token = "123abc" + include_submodules = true }, { type = "pull_from_s3", diff --git a/internal/api/deployments.go b/internal/api/deployments.go index 32cac634..5980a29e 100644 --- a/internal/api/deployments.go +++ b/internal/api/deployments.go @@ -105,34 +105,19 @@ type GlobalConcurrencyLimit struct { // SlotDecayPerSecond int `json:"slot_decay_per_second"` } -// PullStep contains instructions for preparing your flows for a deployment run. -type PullStep struct { - // Type is the type of pull step. - // One of: - // - set_working_directory - // - git_clone - // - pull_from_azure_blob_storage - // - pull_from_gcs - // - pull_from_s3 - Type string `json:"type"` - +// PullStepCommon is a representation of the common fields for certain pull steps. +type PullStepCommon struct { // Credentials is the credentials to use for the pull step. // Used on all PullStep types. Credentials *string `json:"credentials,omitempty"` // Requires is a list of Python package dependencies. Requires *string `json:"requires,omitempty"` +} - // - // Fields for set_working_directory - // - - // The directory to set as the working directory. - Directory *string `json:"directory,omitempty"` - - // - // Fields for git_clone - // +// PullStepGitClone is a representation of a pull step that clones a git repository. +type PullStepGitClone struct { + PullStepCommon // The URL of the repository to clone. Repository *string `json:"repository,omitempty"` @@ -143,9 +128,19 @@ type PullStep struct { // Access token for the repository. AccessToken *string `json:"access_token,omitempty"` - // - // Fields for pull_from_{cloud} - // + // IncludeSubmodules determines whether to include submodules when cloning the repository. + IncludeSubmodules *bool `json:"include_submodules,omitempty"` +} + +// PullStepSetWorkingDirectory is a representation of a pull step that sets the working directory. +type PullStepSetWorkingDirectory struct { + // The directory to set as the working directory. + Directory *string `json:"directory,omitempty"` +} + +// PullStepPullFrom is a representation of a pull step that pulls from a remote storage bucket. +type PullStepPullFrom struct { + PullStepCommon // The name of the bucket where files are stored. Bucket *string `json:"bucket,omitempty"` @@ -153,3 +148,12 @@ type PullStep struct { // The folder in the bucket where files are stored. Folder *string `json:"folder,omitempty"` } + +// PullStep contains instructions for preparing your flows for a deployment run. +type PullStep struct { + PullStepGitClone *PullStepGitClone `json:"prefect.deployments.steps.git_clone,omitempty"` + PullStepSetWorkingDirectory *PullStepSetWorkingDirectory `json:"prefect.deployments.steps.set_working_directory,omitempty"` + PullStepPullFromAzureBlobStorage *PullStepPullFrom `json:"prefect_azure.deployments.steps.pull_from_azure_blob_storage,omitempty"` + PullStepPullFromGCS *PullStepPullFrom `json:"prefect_gcp.deployments.steps.pull_from_gcs,omitempty"` + PullStepPullFromS3 *PullStepPullFrom `json:"prefect_aws.deployments.steps.pull_from_s3,omitempty"` +} diff --git a/internal/provider/datasources/deployment.go b/internal/provider/datasources/deployment.go index 530ce0c7..3788d049 100644 --- a/internal/provider/datasources/deployment.go +++ b/internal/provider/datasources/deployment.go @@ -205,6 +205,10 @@ The Deployment ID takes precedence over deployment name. Computed: true, Description: "(For type 'git_clone') Access token for the repository. Refer to a credentials block for security purposes. Used in leiu of 'credentials'.", }, + "include_submodules": schema.BoolAttribute{ + Computed: true, + Description: "(For type 'git_clone') Whether to include submodules when cloning the repository.", + }, "bucket": schema.StringAttribute{ Computed: true, Description: "(For type 'pull_from_*') The name of the bucket where files are stored.", diff --git a/internal/provider/resources/deployment.go b/internal/provider/resources/deployment.go index 87751e98..61b8d416 100644 --- a/internal/provider/resources/deployment.go +++ b/internal/provider/resources/deployment.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" "github.com/hashicorp/terraform-plugin-framework-jsontypes/jsontypes" + "github.com/hashicorp/terraform-plugin-framework-validators/boolvalidator" "github.com/hashicorp/terraform-plugin-framework-validators/int64validator" "github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator" "github.com/hashicorp/terraform-plugin-framework/attr" @@ -111,6 +112,9 @@ type PullStepModel struct { // Access token for the repository. AccessToken types.String `tfsdk:"access_token"` + // IncludeSubmodules is whether to include submodules in the clone. + IncludeSubmodules types.Bool `tfsdk:"include_submodules"` + // // Fields for pull_from_{cloud} // @@ -349,15 +353,16 @@ func (r *DeploymentResource) Schema(_ context.Context, _ resource.SchemaRequest, Default: listdefault.StaticValue(basetypes.NewListValueMust( types.ObjectType{ AttrTypes: map[string]attr.Type{ - "type": types.StringType, - "credentials": types.StringType, - "requires": types.StringType, - "directory": types.StringType, - "repository": types.StringType, - "branch": types.StringType, - "access_token": types.StringType, - "bucket": types.StringType, - "folder": types.StringType, + "type": types.StringType, + "credentials": types.StringType, + "requires": types.StringType, + "directory": types.StringType, + "repository": types.StringType, + "branch": types.StringType, + "access_token": types.StringType, + "bucket": types.StringType, + "folder": types.StringType, + "include_submodules": types.BoolType, }, }, []attr.Value{}, @@ -388,32 +393,39 @@ func (r *DeploymentResource) Schema(_ context.Context, _ resource.SchemaRequest, "directory": schema.StringAttribute{ Description: "(For type 'set_working_directory') The directory to set as the working directory.", Optional: true, - Validators: validatorsForConflictingAttributes(nonDirectoryAttributes), + Validators: []validator.String{ + stringvalidator.ConflictsWith(pathExpressionsForAttributes(nonDirectoryAttributes)...), + }, }, "repository": schema.StringAttribute{ Description: "(For type 'git_clone') The URL of the repository to clone.", Optional: true, - Validators: validatorsForConflictingAttributes(nonGitCloneAttributes), + Validators: stringConflictsWithValidators(nonGitCloneAttributes), }, "branch": schema.StringAttribute{ Description: "(For type 'git_clone') The branch to clone. If not provided, the default branch is used.", Optional: true, - Validators: validatorsForConflictingAttributes(nonGitCloneAttributes), + Validators: stringConflictsWithValidators(nonGitCloneAttributes), }, "access_token": schema.StringAttribute{ Description: "(For type 'git_clone') Access token for the repository. Refer to a credentials block for security purposes. Used in leiu of 'credentials'.", Optional: true, - Validators: validatorsForConflictingAttributes(nonGitCloneAttributes), + Validators: stringConflictsWithValidators(nonGitCloneAttributes), + }, + "include_submodules": schema.BoolAttribute{ + Description: "(For type 'git_clone') Whether to include submodules when cloning the repository.", + Optional: true, + Validators: boolConflictsWithValidators(nonGitCloneAttributes), }, "bucket": schema.StringAttribute{ Description: "(For type 'pull_from_*') The name of the bucket where files are stored.", Optional: true, - Validators: validatorsForConflictingAttributes(nonPullFromAttributes), + Validators: stringConflictsWithValidators(nonPullFromAttributes), }, "folder": schema.StringAttribute{ Description: "(For type 'pull_from_*') The folder in the bucket where files are stored.", Optional: true, - Validators: validatorsForConflictingAttributes(nonPullFromAttributes), + Validators: stringConflictsWithValidators(nonPullFromAttributes), }, }, }, @@ -430,16 +442,43 @@ func mapPullStepsTerraformToAPI(tfPullSteps []PullStepModel) ([]api.PullStep, di for i := range tfPullSteps { tfPullStep := tfPullSteps[i] - apiPullStep := api.PullStep{ - Type: tfPullStep.Type.ValueString(), + pullStepCommon := api.PullStepCommon{ Credentials: tfPullStep.Credentials.ValueStringPointer(), Requires: tfPullStep.Requires.ValueStringPointer(), - Directory: tfPullStep.Directory.ValueStringPointer(), - Repository: tfPullStep.Repository.ValueStringPointer(), - Branch: tfPullStep.Branch.ValueStringPointer(), - AccessToken: tfPullStep.AccessToken.ValueStringPointer(), - Bucket: tfPullStep.Bucket.ValueStringPointer(), - Folder: tfPullStep.Folder.ValueStringPointer(), + } + + // Steps that pull from remote storage have the same fields. + // Define the struct here for reuse in each of those cases. + pullStepPullFrom := api.PullStepPullFrom{ + PullStepCommon: pullStepCommon, + Bucket: tfPullStep.Bucket.ValueStringPointer(), + Folder: tfPullStep.Folder.ValueStringPointer(), + } + + var apiPullStep api.PullStep + switch tfPullStep.Type.ValueString() { + case "git_clone": + apiPullStep.PullStepGitClone = &api.PullStepGitClone{ + PullStepCommon: pullStepCommon, + Repository: tfPullStep.Repository.ValueStringPointer(), + Branch: tfPullStep.Branch.ValueStringPointer(), + AccessToken: tfPullStep.AccessToken.ValueStringPointer(), + IncludeSubmodules: tfPullStep.IncludeSubmodules.ValueBoolPointer(), + } + + case "set_working_directory": + apiPullStep.PullStepSetWorkingDirectory = &api.PullStepSetWorkingDirectory{ + Directory: tfPullStep.Directory.ValueStringPointer(), + } + + case "pull_from_azure_blob_storage": + apiPullStep.PullStepPullFromAzureBlobStorage = &pullStepPullFrom + + case "pull_from_gcs": + apiPullStep.PullStepPullFromGCS = &pullStepPullFrom + + case "pull_from_s3": + apiPullStep.PullStepPullFromS3 = &pullStepPullFrom } pullSteps = append(pullSteps, apiPullStep) @@ -456,16 +495,60 @@ func mapPullStepsAPIToTerraform(pullSteps []api.PullStep) ([]PullStepModel, diag for i := range pullSteps { pullStep := pullSteps[i] - pullStepModel := PullStepModel{ - Type: types.StringValue(pullStep.Type), - Credentials: types.StringPointerValue(pullStep.Credentials), - Requires: types.StringPointerValue(pullStep.Requires), - Directory: types.StringPointerValue(pullStep.Directory), - Repository: types.StringPointerValue(pullStep.Repository), - Branch: types.StringPointerValue(pullStep.Branch), - AccessToken: types.StringPointerValue(pullStep.AccessToken), - Bucket: types.StringPointerValue(pullStep.Bucket), - Folder: types.StringPointerValue(pullStep.Folder), + var pullStepModel PullStepModel + + // PullStepGitClone + if pullStep.PullStepGitClone != nil { + pullStepModel.Type = types.StringValue("git_clone") + pullStepModel.Repository = types.StringPointerValue(pullStep.PullStepGitClone.Repository) + pullStepModel.Branch = types.StringPointerValue(pullStep.PullStepGitClone.Branch) + pullStepModel.AccessToken = types.StringPointerValue(pullStep.PullStepGitClone.AccessToken) + pullStepModel.IncludeSubmodules = types.BoolPointerValue(pullStep.PullStepGitClone.IncludeSubmodules) + + // common fields + pullStepModel.Credentials = types.StringPointerValue(pullStep.PullStepGitClone.Credentials) + pullStepModel.Requires = types.StringPointerValue(pullStep.PullStepGitClone.Requires) + } + + // PullStepSetWorkingDirectory + if pullStep.PullStepSetWorkingDirectory != nil { + pullStepModel.Type = types.StringValue("set_working_directory") + pullStepModel.Directory = types.StringValue(*pullStep.PullStepSetWorkingDirectory.Directory) + + // common fields not used on this pull step type + } + + // PullStepPullFromAzureBlobStorage + if pullStep.PullStepPullFromAzureBlobStorage != nil { + pullStepModel.Type = types.StringValue("pull_from_azure_blob_storage") + pullStepModel.Bucket = types.StringPointerValue(pullStep.PullStepPullFromAzureBlobStorage.Bucket) + pullStepModel.Folder = types.StringPointerValue(pullStep.PullStepPullFromAzureBlobStorage.Folder) + + // common fields + pullStepModel.Credentials = types.StringPointerValue(pullStep.PullStepPullFromAzureBlobStorage.Credentials) + pullStepModel.Requires = types.StringPointerValue(pullStep.PullStepPullFromAzureBlobStorage.Requires) + } + + // PullStepPullFromGCS + if pullStep.PullStepPullFromGCS != nil { + pullStepModel.Type = types.StringValue("pull_from_gcs") + pullStepModel.Bucket = types.StringPointerValue(pullStep.PullStepPullFromGCS.Bucket) + pullStepModel.Folder = types.StringPointerValue(pullStep.PullStepPullFromGCS.Folder) + + // common fields + pullStepModel.Credentials = types.StringPointerValue(pullStep.PullStepPullFromGCS.Credentials) + pullStepModel.Requires = types.StringPointerValue(pullStep.PullStepPullFromGCS.Requires) + } + + // PullStepPullFromS3 + if pullStep.PullStepPullFromS3 != nil { + pullStepModel.Type = types.StringValue("pull_from_s3") + pullStepModel.Bucket = types.StringPointerValue(pullStep.PullStepPullFromS3.Bucket) + pullStepModel.Folder = types.StringPointerValue(pullStep.PullStepPullFromS3.Folder) + + // common fields + pullStepModel.Credentials = types.StringPointerValue(pullStep.PullStepPullFromS3.Credentials) + pullStepModel.Requires = types.StringPointerValue(pullStep.PullStepPullFromS3.Requires) } tfPullStepsModel = append(tfPullStepsModel, pullStepModel) @@ -911,7 +994,7 @@ func (r *DeploymentResource) ImportState(ctx context.Context, req resource.Impor } } -// validatorsForConflictingAttributes provides a list of string validators +// pathExpressionsForAttributes provides a list of path expressions // used in a ConflictsWith validator for a specific attribute. // // This approach is used in lieu of a ConfigValidators method because we take @@ -922,15 +1005,29 @@ func (r *DeploymentResource) ImportState(ctx context.Context, req resource.Impor // be more concise when defining the conflicting attributes. Defining them in // ConfigValidators instead would be much more verbose, and disconnected from // the source of truth. -func validatorsForConflictingAttributes(attributes []string) []validator.String { +func pathExpressionsForAttributes(attributes []string) []path.Expression { pathExpressions := make([]path.Expression, 0) for _, key := range attributes { pathExpressions = append(pathExpressions, path.MatchRelative().AtParent().AtName(key)) } + return pathExpressions +} + +// stringConflictsWithValidators provides a list of string validators +// for a specific attribute, allowing for more concise schema definitions. +func stringConflictsWithValidators(attributes []string) []validator.String { return []validator.String{ - stringvalidator.ConflictsWith(pathExpressions...), + stringvalidator.ConflictsWith(pathExpressionsForAttributes(attributes)...), + } +} + +// boolConflictsWithValidators provides a list of bool validators +// for a specific attribute, allowing for more concise schema definitions. +func boolConflictsWithValidators(attributes []string) []validator.Bool { + return []validator.Bool{ + boolvalidator.ConflictsWith(pathExpressionsForAttributes(attributes)...), } } @@ -943,6 +1040,7 @@ var ( "repository", "branch", "access_token", + "include_submodules", } pullFromAttributes = []string{ diff --git a/internal/provider/resources/deployment_test.go b/internal/provider/resources/deployment_test.go index 9b5ea9d5..3432be5a 100644 --- a/internal/provider/resources/deployment_test.go +++ b/internal/provider/resources/deployment_test.go @@ -101,33 +101,80 @@ resource "prefect_deployment" "{{.DeploymentName}}" { work_queue_name = "{{.WorkQueueName}}" parameter_openapi_schema = jsonencode({{.ParameterOpenAPISchema}}) pull_steps = [ - {{range .PullSteps}} + {{range .PullSteps}} { - type = "{{.Type}}" - - {{- if .Directory }} + {{- with .PullStepSetWorkingDirectory }} + type = "set_working_directory" directory = "{{.Directory}}" - {{- end }} + {{- end}} - {{- if .Repository }} + {{- with .PullStepGitClone }} + type = "git_clone" repository = "{{.Repository}}" - {{- end }} - - {{- if .Branch }} + {{- if .Branch }} branch = "{{.Branch}}" + {{- end }} + {{- if .AccessToken }} + access_token = "{{.AccessToken}}" + {{- end }} + {{- if .IncludeSubmodules }} + include_submodules = {{.IncludeSubmodules}} + {{- end }} + {{- if .Credentials }} + credentials = "{{.Credentials}}" + {{- end }} + {{- if .Requires }} + requires = "{{.Requires}}" + {{- end }} {{- end }} - {{- if .AccessToken }} - access_token = "{{.AccessToken}}" + {{- with .PullStepPullFromAzureBlobStorage }} + type = "pull_from_azure_blob_storage" + {{- if .Bucket }} + bucket = "{{.Bucket}}" + {{- end}} + {{- if .Folder }} + folder = "{{.Folder}}" + {{- end}} + {{- if .Credentials }} + credentials = "{{.Credentials}}" + {{- end }} + {{- if .Requires }} + requires = "{{.Requires}}" + {{- end }} {{- end }} - {{- if .Bucket }} + {{- with .PullStepPullFromGCS }} + type = "pull_from_gcs" + {{- if .Bucket }} bucket = "{{.Bucket}}" - {{- end}} + {{- end}} + {{- if .Folder }} + folder = "{{.Folder}}" + {{- end}} + {{- if .Credentials }} + credentials = "{{.Credentials}}" + {{- end }} + {{- if .Requires }} + requires = "{{.Requires}}" + {{- end }} + {{- end }} - {{- if .Folder }} + {{- with .PullStepPullFromS3 }} + type = "pull_from_s3" + {{- if .Bucket }} + bucket = "{{.Bucket}}" + {{- end}} + {{- if .Folder }} folder = "{{.Folder}}" - {{- end}} + {{- end}} + {{- if .Credentials }} + credentials = "{{.Credentials}}" + {{- end }} + {{- if .Requires }} + requires = "{{.Requires}}" + {{- end }} + {{- end }} }, {{end}} ] @@ -169,8 +216,9 @@ func TestAccResource_deployment(t *testing.T) { Paused: false, PullSteps: []api.PullStep{ { - Type: "set_working_directory", - Directory: ptr.To("/some/directory"), + PullStepSetWorkingDirectory: &api.PullStepSetWorkingDirectory{ + Directory: ptr.To("/some/directory"), + }, }, }, Tags: []string{"test1", "test2"}, @@ -226,19 +274,27 @@ func TestAccResource_deployment(t *testing.T) { // PullSteps require a replacement of the resource. PullSteps: []api.PullStep{ { - Type: "set_working_directory", - Directory: ptr.To("/some/directory"), + PullStepSetWorkingDirectory: &api.PullStepSetWorkingDirectory{ + Directory: ptr.To("/some/other/directory"), + }, }, { - Type: "git_clone", - Repository: ptr.To("https://github.com/prefecthq/prefect"), - Branch: ptr.To("main"), - AccessToken: ptr.To("123abc"), + PullStepGitClone: &api.PullStepGitClone{ + Repository: ptr.To("https://github.com/prefecthq/prefect"), + Branch: ptr.To("main"), + AccessToken: ptr.To("123abc"), + IncludeSubmodules: ptr.To(true), + }, }, { - Type: "pull_from_s3", - Bucket: ptr.To("some-bucket"), - Folder: ptr.To("some-folder"), + PullStepPullFromS3: &api.PullStepPullFrom{ + Bucket: ptr.To("some-bucket"), + Folder: ptr.To("some-folder"), + PullStepCommon: api.PullStepCommon{ + Credentials: ptr.To("some-credentials"), + Requires: ptr.To("prefect-aws>=0.3.4"), + }, + }, }, }, @@ -380,7 +436,7 @@ func testAccCheckDeploymentValues(fetchedDeployment *api.Deployment, expectedVal } if !reflect.DeepEqual(fetchedDeployment.PullSteps, expectedValues.pullSteps) { - return fmt.Errorf("Expected pull steps to be %v, got %v", expectedValues.pullSteps, fetchedDeployment.PullSteps) + return fmt.Errorf("Expected pull steps to be: \n%v\n got \n%v", expectedValues.pullSteps, fetchedDeployment.PullSteps) } return nil