Skip to content

Commit

Permalink
Merge pull request #71 from tomarv2/develop
Browse files Browse the repository at this point in the history
Pulling refs/heads/develop into main
tomarv2 authored Sep 13, 2022

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents d526d14 + eda41d1 commit f28fd5c
Showing 5 changed files with 42 additions and 15 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@
## Versions

- Module tested for Terraform 1.0.1.
- `databricks/databricks` provider version [0.5.7](https://registry.terraform.io/providers/databricks/databricks/latest)
- `databricks/databricks` provider version [1.3.0](https://registry.terraform.io/providers/databricks/databricks/latest)
- AWS provider version [4.14](https://registry.terraform.io/providers/hashicorp/aws/latest).
- `main` branch: Provider versions not pinned to keep up with Terraform releases.
- `tags` releases: Tags are pinned with versions (use <a href="https://github.com/tomarv2/terraform-databricks-workspace-management/tags" alt="GitHub tag">
33 changes: 30 additions & 3 deletions examples/jobs/4_job_on_existing_cluster_with_new_notebooks/main.tf
Original file line number Diff line number Diff line change
@@ -12,16 +12,36 @@ terraform {
}
}

module "cluster" {
source = "../../../"
# ------------------------------------------------
# CLUSTER
# ------------------------------------------------
deploy_cluster = true
fixed_value = 0

# libraries = {
# maven = {
# "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21" = {}
# }
# python_wheel = {}
# }

module "databricks_workspace_management" {
# ------------------------------------------------
# Do not change the teamid, prjid once set.
teamid = var.teamid
prjid = var.prjid
}

module "jobs" {
source = "../../../"

# ------------------------------------------------
# JOB
# ------------------------------------------------
deploy_jobs = true
cluster_id = "0907-052446-bike152"
fixed_value = 1
cluster_id = module.cluster.cluster_id
#fixed_value = 1
retry_on_timeout = false
max_retries = 3
timeout = 30
@@ -57,6 +77,13 @@ module "databricks_workspace_management" {
local_path = "notebooks/sample2.py"
}
]
jobs_access_control = [
{
group_name = "demo"
permission_level = "CAN_MANAGE_RUN"
}
]
depends_on = [module.cluster]
# ------------------------------------------------
# Do not change the teamid, prjid once set.
teamid = var.teamid
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
output "notebook_url" {
description = "databricks notebook url"
value = module.databricks_workspace_management.notebook_url
value = module.jobs.notebook_url
}

output "job_url" {
description = "databricks job url"
value = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_job
value = module.jobs.existing_cluster_new_job_new_notebooks_job
}

output "job_id" {
description = "databricks job id"
value = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_id
value = module.jobs.existing_cluster_new_job_new_notebooks_id
}

8 changes: 4 additions & 4 deletions job.tf
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@
# 1. NEW CLUSTER WITH NEW NOTEBOOKS
# ------------------------------------------------
resource "databricks_job" "new_cluster_new_job_new_notebooks" {
for_each = (var.deploy_jobs == true && var.cluster_id == null && var.deploy_cluster == true && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
for_each = (var.deploy_jobs == true && var.deploy_cluster == true && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}

name = "${each.value.job_name} (Terraform managed)"

@@ -93,7 +93,7 @@ resource "databricks_job" "new_cluster_new_job_new_notebooks" {
# 2. NEW CLUSTER WITH EXITING NOTEBOOKS
# ------------------------------------------------
resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
for_each = (var.deploy_jobs == true && var.cluster_id == null && var.deploy_cluster == true && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
for_each = (var.deploy_jobs == true && var.deploy_cluster == true && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}

name = "${each.value.job_name} (Terraform managed)"

@@ -184,7 +184,7 @@ resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
# 3. EXISTING CLUSTER WITH NEW NOTEBOOKS
# ------------------------------------------------
resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
for_each = (var.deploy_jobs == true && (var.cluster_id != null || var.deploy_cluster == false) && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
for_each = (var.deploy_jobs == true && (var.deploy_cluster == false) && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}

name = "${each.value.job_name} (Terraform managed)"
existing_cluster_id = local.cluster_info
@@ -224,7 +224,7 @@ resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
# 4. EXISTING CLUSTER WITH EXITING NOTEBOOKS
# ------------------------------------------------
resource "databricks_job" "existing_cluster_new_job_existing_notebooks" {
for_each = var.deploy_jobs == true && (var.cluster_id != null || var.deploy_cluster == false) && var.remote_notebooks != null ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
for_each = var.deploy_jobs == true && (var.deploy_cluster == false) && var.remote_notebooks != null ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}

name = "${each.value.job_name} (Terraform managed)"
existing_cluster_id = local.cluster_info
8 changes: 4 additions & 4 deletions permissions.tf
Original file line number Diff line number Diff line change
@@ -72,7 +72,7 @@ resource "databricks_permissions" "policy" {
# 1. NEW CLUSTER WITH NEW NOTEBOOKS
# ------------------------------------------------
resource "databricks_permissions" "new_cluster_new_job_new_notebooks" {
for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.cluster_id == null && var.local_notebooks != null) ? databricks_job.new_cluster_new_job_new_notebooks : {}
for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.local_notebooks != null) ? databricks_job.new_cluster_new_job_new_notebooks : {}

job_id = each.value.id

@@ -89,7 +89,7 @@ resource "databricks_permissions" "new_cluster_new_job_new_notebooks" {
# 2. EXISTING CLUSTER WITH NEW NOTEBOOKS
# ------------------------------------------------
resource "databricks_permissions" "existing_cluster_new_job_new_notebooks" {
for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.cluster_id != null && var.local_notebooks != null) ? databricks_job.existing_cluster_new_job_new_notebooks : {}
for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.local_notebooks != null) ? databricks_job.existing_cluster_new_job_new_notebooks : {}

job_id = each.value.id

@@ -106,7 +106,7 @@ resource "databricks_permissions" "existing_cluster_new_job_new_notebooks" {
# 3. NEW CLUSTER WITH EXITING NOTEBOOKS
# ------------------------------------------------
resource "databricks_permissions" "new_cluster_new_job_existing_notebooks" {
for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.cluster_id == null && var.remote_notebooks != null) ? databricks_job.new_cluster_new_job_existing_notebooks : {}
for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.remote_notebooks != null) ? databricks_job.new_cluster_new_job_existing_notebooks : {}

job_id = each.value.id

@@ -123,7 +123,7 @@ resource "databricks_permissions" "new_cluster_new_job_existing_notebooks" {
# 4. EXISTING CLUSTER WITH EXITING NOTEBOOKS
# ------------------------------------------------
resource "databricks_permissions" "existing_cluster_new_job_existing_notebooks" {
for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.cluster_id != null && var.remote_notebooks != null) ? databricks_job.existing_cluster_new_job_existing_notebooks : {}
for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.remote_notebooks != null) ? databricks_job.existing_cluster_new_job_existing_notebooks : {}

job_id = each.value.id

0 comments on commit f28fd5c

Please sign in to comment.