-
Notifications
You must be signed in to change notification settings - Fork 72
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Loading status checks…
Added E2E test to run python wheels on interactive cluster created in…
… bundle
1 parent
11f75fd
commit 259d1f7
Showing
6 changed files
with
102 additions
and
4 deletions.
There are no files selected for viewing
25 changes: 25 additions & 0 deletions
25
internal/bundle/bundles/python_wheel_task_with_cluster/databricks_template_schema.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
{ | ||
"properties": { | ||
"project_name": { | ||
"type": "string", | ||
"default": "my_test_code", | ||
"description": "Unique name for this project" | ||
}, | ||
"spark_version": { | ||
"type": "string", | ||
"description": "Spark version used for job cluster" | ||
}, | ||
"node_type_id": { | ||
"type": "string", | ||
"description": "Node type id for job cluster" | ||
}, | ||
"unique_id": { | ||
"type": "string", | ||
"description": "Unique ID for job name" | ||
}, | ||
"instance_pool_id": { | ||
"type": "string", | ||
"description": "Instance pool id for job cluster" | ||
} | ||
} | ||
} |
29 changes: 29 additions & 0 deletions
29
internal/bundle/bundles/python_wheel_task_with_cluster/template/databricks.yml.tmpl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
bundle: | ||
name: wheel-task | ||
|
||
workspace: | ||
root_path: "~/.bundle/{{.unique_id}}" | ||
|
||
resources: | ||
clusters: | ||
test_cluster: | ||
cluster_name: "test-cluster-{{.unique_id}}" | ||
spark_version: "{{.spark_version}}" | ||
node_type_id: "{{.node_type_id}}" | ||
num_workers: 1 | ||
data_security_mode: USER_ISOLATION | ||
|
||
jobs: | ||
some_other_job: | ||
name: "[${bundle.target}] Test Wheel Job {{.unique_id}}" | ||
tasks: | ||
- task_key: TestTask | ||
existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" | ||
python_wheel_task: | ||
package_name: my_test_code | ||
entry_point: run | ||
parameters: | ||
- "one" | ||
- "two" | ||
libraries: | ||
- whl: ./dist/*.whl |
15 changes: 15 additions & 0 deletions
15
internal/bundle/bundles/python_wheel_task_with_cluster/template/setup.py.tmpl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
from setuptools import setup, find_packages | ||
|
||
import {{.project_name}} | ||
|
||
setup( | ||
name="{{.project_name}}", | ||
version={{.project_name}}.__version__, | ||
author={{.project_name}}.__author__, | ||
url="https://databricks.com", | ||
author_email="[email protected]", | ||
description="my example wheel", | ||
packages=find_packages(include=["{{.project_name}}"]), | ||
entry_points={"group1": "run={{.project_name}}.__main__:main"}, | ||
install_requires=["setuptools"], | ||
) |
2 changes: 2 additions & 0 deletions
2
...rnal/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
__version__ = "0.0.1" | ||
__author__ = "Databricks" |
16 changes: 16 additions & 0 deletions
16
...rnal/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__main__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
""" | ||
The entry point of the Python Wheel | ||
""" | ||
|
||
import sys | ||
|
||
|
||
def main(): | ||
# This method will print the provided arguments | ||
print("Hello from my func") | ||
print("Got arguments:") | ||
print(sys.argv) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters