From f0ee7ea36bd1eecc1bbdfce0bac57a813cbca577 Mon Sep 17 00:00:00 2001 From: Emil Christensen Date: Mon, 7 Oct 2024 11:43:41 -0400 Subject: [PATCH 1/6] Adds cloud run v2 to worker_metadata --- docs/data-sources/worker_metadata.md | 2 ++ internal/provider/datasources/worker_metadata.go | 14 ++++++++++++++ .../provider/datasources/worker_metadata_test.go | 2 ++ 3 files changed, 18 insertions(+) diff --git a/docs/data-sources/worker_metadata.md b/docs/data-sources/worker_metadata.md index a0dc4fe2..d08b85e5 100644 --- a/docs/data-sources/worker_metadata.md +++ b/docs/data-sources/worker_metadata.md @@ -63,6 +63,8 @@ Read-Only: - `azure_container_instances_push` (String) Default base job configuration for Azure Container Instances Push workers - `cloud_run` (String) Default base job configuration for Cloud Run workers - `cloud_run_push` (String) Default base job configuration for Cloud Run Push workers +- `cloud_run_v2` (String) Default base job configuration for Cloud Run V2 workers +- `cloud_run_v2_push` (String) Default base job configuration for Cloud Run V2 Push workers - `docker` (String) Default base job configuration for Docker workers - `ecs` (String) Default base job configuration for ECS workers - `ecs_push` (String) Default base job configuration for ECS Push workers diff --git a/internal/provider/datasources/worker_metadata.go b/internal/provider/datasources/worker_metadata.go index 76d2c115..16b7a0b7 100644 --- a/internal/provider/datasources/worker_metadata.go +++ b/internal/provider/datasources/worker_metadata.go @@ -87,6 +87,11 @@ Use this data source to get the default base job configurations for those common Description: "Default base job configuration for Cloud Run workers", CustomType: jsontypes.NormalizedType{}, }, + "cloud_run_v2": schema.StringAttribute{ + Computed: true, + Description: "Default base job configuration for Cloud Run V2 workers", + CustomType: jsontypes.NormalizedType{}, + }, "vertex_ai": schema.StringAttribute{ Computed: true, Description: "Default base job configuration for Vertex AI workers", @@ -112,6 +117,11 @@ Use this data source to get the default base job configurations for those common Description: "Default base job configuration for Cloud Run Push workers", CustomType: jsontypes.NormalizedType{}, }, + "cloud_run_v2_push": schema.StringAttribute{ + Computed: true, + Description: "Default base job configuration for Cloud Run V2 Push workers", + CustomType: jsontypes.NormalizedType{}, + }, "ecs_push": schema.StringAttribute{ Computed: true, Description: "Default base job configuration for ECS Push workers", @@ -163,11 +173,13 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read "azure_container_instances": jsontypes.NormalizedType{}, "docker": jsontypes.NormalizedType{}, "cloud_run": jsontypes.NormalizedType{}, + "cloud_run_v2": jsontypes.NormalizedType{}, "vertex_ai": jsontypes.NormalizedType{}, "prefect_agent": jsontypes.NormalizedType{}, "process": jsontypes.NormalizedType{}, "azure_container_instances_push": jsontypes.NormalizedType{}, "cloud_run_push": jsontypes.NormalizedType{}, + "cloud_run_v2_push": jsontypes.NormalizedType{}, "ecs_push": jsontypes.NormalizedType{}, } attributeValues := map[string]attr.Value{ @@ -176,11 +188,13 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read "azure_container_instances": jsontypes.NewNormalizedValue(string(remap["azure-container-instance"])), "docker": jsontypes.NewNormalizedValue(string(remap["docker"])), "cloud_run": jsontypes.NewNormalizedValue(string(remap["cloud-run"])), + "cloud_run_v2": jsontypes.NewNormalizedValue(string(remap["cloud-run-v2"])), "vertex_ai": jsontypes.NewNormalizedValue(string(remap["vertex-ai"])), "prefect_agent": jsontypes.NewNormalizedValue(string(remap["prefect-agent"])), "process": jsontypes.NewNormalizedValue(string(remap["process"])), "azure_container_instances_push": jsontypes.NewNormalizedValue(string(remap["azure-container-instance:push"])), "cloud_run_push": jsontypes.NewNormalizedValue(string(remap["cloud-run:push"])), + "cloud_run_v2_push": jsontypes.NewNormalizedValue(string(remap["cloud-run-v2:push"])), "ecs_push": jsontypes.NewNormalizedValue(string(remap["ecs:push"])), } diff --git a/internal/provider/datasources/worker_metadata_test.go b/internal/provider/datasources/worker_metadata_test.go index 1be035f5..9f97c50a 100644 --- a/internal/provider/datasources/worker_metadata_test.go +++ b/internal/provider/datasources/worker_metadata_test.go @@ -30,11 +30,13 @@ func TestAccDatasource_worker_metadata(t *testing.T) { resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.azure_container_instances"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.docker"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run"), + resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_v2"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.vertex_ai"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.prefect_agent"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.process"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.azure_container_instances_push"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_push"), + resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_v2_push"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.ecs_push"), ), }, From ec285bcee8a4bc10291b7ece929a413d6d3231f0 Mon Sep 17 00:00:00 2001 From: Emil Christensen Date: Mon, 7 Oct 2024 12:19:04 -0400 Subject: [PATCH 2/6] Fixes test in TestAccDatasource_worker_metadata --- internal/provider/datasources/worker_metadata_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/provider/datasources/worker_metadata_test.go b/internal/provider/datasources/worker_metadata_test.go index 9f97c50a..b1c4891b 100644 --- a/internal/provider/datasources/worker_metadata_test.go +++ b/internal/provider/datasources/worker_metadata_test.go @@ -24,7 +24,7 @@ func TestAccDatasource_worker_metadata(t *testing.T) { { Config: fixtureAccWorkerMetadtata(), Check: resource.ComposeAggregateTestCheckFunc( - resource.TestCheckResourceAttr(datasourceName, "base_job_configs.%", "11"), + resource.TestCheckResourceAttr(datasourceName, "base_job_configs.%", "13"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.kubernetes"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.ecs"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.azure_container_instances"), From 3e2883657dcacdd396a51bb5cce5c85f592876f9 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Tue, 8 Oct 2024 12:29:35 -0500 Subject: [PATCH 3/6] Use the same endpoints as the Prefect client The endpoint to list the base job configs differs between OSS and Cloud. This change calculates the correct endpoint, and updates the test to refer to the specific evergreen workspace used for testing. --- internal/api/client.go | 2 +- internal/client/collections.go | 27 ++++++++++++++++--- .../provider/datasources/worker_metadata.go | 16 ++++++++++- .../datasources/worker_metadata_test.go | 17 +++++++++--- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index 3e998da4..049fc65f 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -12,7 +12,7 @@ type PrefectClient interface { BlockDocuments(accountID uuid.UUID, workspaceID uuid.UUID) (BlockDocumentClient, error) BlockSchemas(accountID uuid.UUID, workspaceID uuid.UUID) (BlockSchemaClient, error) BlockTypes(accountID uuid.UUID, workspaceID uuid.UUID) (BlockTypeClient, error) - Collections() (CollectionsClient, error) + Collections(accountID uuid.UUID, workspaceID uuid.UUID) (CollectionsClient, error) Deployments(accountID uuid.UUID, workspaceID uuid.UUID) (DeploymentsClient, error) Teams(accountID uuid.UUID) (TeamsClient, error) Flows(accountID uuid.UUID, workspaceID uuid.UUID) (FlowsClient, error) diff --git a/internal/client/collections.go b/internal/client/collections.go index 4886a0fa..f6ebe6de 100644 --- a/internal/client/collections.go +++ b/internal/client/collections.go @@ -7,7 +7,9 @@ import ( "io" "net/http" + "github.com/google/uuid" "github.com/prefecthq/terraform-provider-prefect/internal/api" + "github.com/prefecthq/terraform-provider-prefect/internal/provider/helpers" ) var _ = api.CollectionsClient(&CollectionsClient{}) @@ -21,18 +23,37 @@ type CollectionsClient struct { // Collections returns an CollectionsClient. // //nolint:ireturn // required to support PrefectClient mocking -func (c *Client) Collections() (api.CollectionsClient, error) { +func (c *Client) Collections(accountID, workspaceID uuid.UUID) (api.CollectionsClient, error) { + if accountID == uuid.Nil { + accountID = c.defaultAccountID + } + + if workspaceID == uuid.Nil { + workspaceID = c.defaultWorkspaceID + } + + if helpers.IsCloudEndpoint(c.endpoint) && (accountID == uuid.Nil || workspaceID == uuid.Nil) { + return nil, fmt.Errorf("prefect Cloud endpoints require an account_id and workspace_id to be set on either the provider or the resource") + } + return &CollectionsClient{ hc: c.hc, apiKey: c.apiKey, - routePrefix: fmt.Sprintf("%s/collections", c.endpoint), + routePrefix: getWorkspaceScopedURL(c.endpoint, accountID, workspaceID, "collections"), }, nil } // GetWorkerMetadataViews returns a map of worker metadata views by prefect package name. // This endpoint serves base job configurations for the primary worker types. func (c *CollectionsClient) GetWorkerMetadataViews(ctx context.Context) (api.WorkerTypeByPackage, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/views/aggregate-worker-metadata", c.routePrefix), http.NoBody) + routeSuffix := "views/aggregate-worker-metadata" + if helpers.IsCloudEndpoint(c.routePrefix) { + routeSuffix = "work_pool_types" + } + + url := fmt.Sprintf("%s/%s", c.routePrefix, routeSuffix) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) if err != nil { return nil, fmt.Errorf("error creating request: %w", err) } diff --git a/internal/provider/datasources/worker_metadata.go b/internal/provider/datasources/worker_metadata.go index 16b7a0b7..2b0e175f 100644 --- a/internal/provider/datasources/worker_metadata.go +++ b/internal/provider/datasources/worker_metadata.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/terraform-plugin-framework/datasource/schema" "github.com/hashicorp/terraform-plugin-framework/types" "github.com/prefecthq/terraform-provider-prefect/internal/api" + "github.com/prefecthq/terraform-provider-prefect/internal/provider/customtypes" "github.com/prefecthq/terraform-provider-prefect/internal/provider/helpers" ) @@ -18,6 +19,9 @@ type WorkerMetadataDataSource struct { } type WorkerMetadataDataSourceModel struct { + AccountID customtypes.UUIDValue `tfsdk:"account_id"` + WorkspaceID customtypes.UUIDValue `tfsdk:"workspace_id"` + BaseJobConfigs types.Object `tfsdk:"base_job_configs"` } @@ -58,6 +62,16 @@ Get metadata information about the common Worker types, such as Kubernetes, ECS, Use this data source to get the default base job configurations for those common Worker types. `, Attributes: map[string]schema.Attribute{ + "account_id": schema.StringAttribute{ + CustomType: customtypes.UUIDType{}, + Description: "Account ID (UUID), defaults to the account set in the provider", + Optional: true, + }, + "workspace_id": schema.StringAttribute{ + CustomType: customtypes.UUIDType{}, + Description: "Workspace ID (UUID), defaults to the workspace set in the provider", + Optional: true, + }, "base_job_configs": schema.SingleNestedAttribute{ Computed: true, Description: "A map of default base job configurations (JSON) for each of the primary worker types", @@ -143,7 +157,7 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read return } - client, err := d.client.Collections() + client, err := d.client.Collections(model.AccountID.ValueUUID(), model.WorkspaceID.ValueUUID()) if err != nil { resp.Diagnostics.Append(helpers.CreateClientErrorDiagnostic("Collections", err)) diff --git a/internal/provider/datasources/worker_metadata_test.go b/internal/provider/datasources/worker_metadata_test.go index b1c4891b..9aeb3176 100644 --- a/internal/provider/datasources/worker_metadata_test.go +++ b/internal/provider/datasources/worker_metadata_test.go @@ -1,6 +1,8 @@ package datasources_test import ( + "fmt" + "os" "testing" "github.com/hashicorp/terraform-plugin-testing/helper/resource" @@ -8,9 +10,18 @@ import ( ) func fixtureAccWorkerMetadtata() string { - return ` -data "prefect_worker_metadata" "default" {} -` + aID := os.Getenv("PREFECT_CLOUD_ACCOUNT_ID") + + return fmt.Sprintf(` +data "prefect_workspace" "evergreen" { + handle = "github-ci-tests" +} + +data "prefect_worker_metadata" "default" { + account_id = "%s" + workspace_id = data.prefect_workspace.evergreen.id +} +`, aID) } //nolint:paralleltest // we use the resource.ParallelTest helper instead From 97db834aaa0d6ac0c46c0dcb266334427989e7c3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 8 Oct 2024 17:49:24 +0000 Subject: [PATCH 4/6] Generate Terraform Docs --- docs/data-sources/worker_metadata.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/data-sources/worker_metadata.md b/docs/data-sources/worker_metadata.md index d08b85e5..3aa54c45 100644 --- a/docs/data-sources/worker_metadata.md +++ b/docs/data-sources/worker_metadata.md @@ -50,6 +50,11 @@ resource "prefect_work_pool" "process" { ## Schema +### Optional + +- `account_id` (String) Account ID (UUID), defaults to the account set in the provider +- `workspace_id` (String) Workspace ID (UUID), defaults to the workspace set in the provider + ### Read-Only - `base_job_configs` (Attributes) A map of default base job configurations (JSON) for each of the primary worker types (see [below for nested schema](#nestedatt--base_job_configs)) From 37f5190dc8d4b45238d74ffff9e53279bc54bf88 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Tue, 8 Oct 2024 16:47:43 -0500 Subject: [PATCH 5/6] Support Modal push workers --- internal/provider/datasources/worker_metadata.go | 8 ++++++++ internal/provider/datasources/worker_metadata_test.go | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/provider/datasources/worker_metadata.go b/internal/provider/datasources/worker_metadata.go index 2b0e175f..31ef7a0a 100644 --- a/internal/provider/datasources/worker_metadata.go +++ b/internal/provider/datasources/worker_metadata.go @@ -141,6 +141,11 @@ Use this data source to get the default base job configurations for those common Description: "Default base job configuration for ECS Push workers", CustomType: jsontypes.NormalizedType{}, }, + "modal_push": schema.StringAttribute{ + Computed: true, + Description: "Default base job configuration for Modal Push workers", + CustomType: jsontypes.NormalizedType{}, + }, }, }, }, @@ -181,6 +186,7 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read } // https://developer.hashicorp.com/terraform/plugin/framework/handling-data/types/object#setting-values + // https://docs.prefect.io/latest/deploy/infrastructure-concepts/work-pools#work-pool-types attributeTypes := map[string]attr.Type{ "kubernetes": jsontypes.NormalizedType{}, "ecs": jsontypes.NormalizedType{}, @@ -195,6 +201,7 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read "cloud_run_push": jsontypes.NormalizedType{}, "cloud_run_v2_push": jsontypes.NormalizedType{}, "ecs_push": jsontypes.NormalizedType{}, + "modal_push": jsontypes.NormalizedType{}, } attributeValues := map[string]attr.Value{ "kubernetes": jsontypes.NewNormalizedValue(string(remap["kubernetes"])), @@ -210,6 +217,7 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read "cloud_run_push": jsontypes.NewNormalizedValue(string(remap["cloud-run:push"])), "cloud_run_v2_push": jsontypes.NewNormalizedValue(string(remap["cloud-run-v2:push"])), "ecs_push": jsontypes.NewNormalizedValue(string(remap["ecs:push"])), + "modal_push": jsontypes.NewNormalizedValue(string(remap["modal:push"])), } obj, diag := types.ObjectValue(attributeTypes, attributeValues) diff --git a/internal/provider/datasources/worker_metadata_test.go b/internal/provider/datasources/worker_metadata_test.go index 9aeb3176..e63b850c 100644 --- a/internal/provider/datasources/worker_metadata_test.go +++ b/internal/provider/datasources/worker_metadata_test.go @@ -35,7 +35,7 @@ func TestAccDatasource_worker_metadata(t *testing.T) { { Config: fixtureAccWorkerMetadtata(), Check: resource.ComposeAggregateTestCheckFunc( - resource.TestCheckResourceAttr(datasourceName, "base_job_configs.%", "13"), + resource.TestCheckResourceAttr(datasourceName, "base_job_configs.%", "14"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.kubernetes"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.ecs"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.azure_container_instances"), @@ -48,6 +48,7 @@ func TestAccDatasource_worker_metadata(t *testing.T) { resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.azure_container_instances_push"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_push"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_v2_push"), + resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.modal_push"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.ecs_push"), ), }, From 70f4382aad096e26a2263fef1f2748ec6808bf58 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 8 Oct 2024 21:50:35 +0000 Subject: [PATCH 6/6] Generate Terraform Docs --- docs/data-sources/worker_metadata.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/data-sources/worker_metadata.md b/docs/data-sources/worker_metadata.md index 3aa54c45..3c19c5bc 100644 --- a/docs/data-sources/worker_metadata.md +++ b/docs/data-sources/worker_metadata.md @@ -74,6 +74,7 @@ Read-Only: - `ecs` (String) Default base job configuration for ECS workers - `ecs_push` (String) Default base job configuration for ECS Push workers - `kubernetes` (String) Default base job configuration for Kubernetes workers +- `modal_push` (String) Default base job configuration for Modal Push workers - `prefect_agent` (String) Default base job configuration for Prefect Agent workers - `process` (String) Default base job configuration for Process workers - `vertex_ai` (String) Default base job configuration for Vertex AI workers