Skip to content

Commit

Permalink
Setup tron secret_volumes in setup_tron_namespace
Browse files Browse the repository at this point in the history
  • Loading branch information
@Molaire
Molaire committed May 31, 2023
1 parent 90678ed commit e9ccd71
Show file tree
Hide file tree
Showing 3 changed files with 125 additions and 1 deletion.
1 change: 1 addition & 0 deletions paasta_tools/tron_tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -869,6 +869,7 @@ def format_tron_action_dict(action_config: TronActionConfig, use_k8s: bool = Fal
"node": action_config.get_node(),
"retries": action_config.get_retries(),
"retries_delay": action_config.get_retries_delay(),
"secret_volumes": action_config.get_secret_volumes(),
"expected_runtime": action_config.get_expected_runtime(),
"trigger_downstreams": action_config.get_trigger_downstreams(),
"triggered_by": action_config.get_triggered_by(),
Expand Down
107 changes: 106 additions & 1 deletion tests/test_tron_tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,38 @@ def test_get_secret_env(self, action_config, test_env, expected_env):
secret_env = action_config.get_secret_env()
assert secret_env == expected_env

@pytest.mark.parametrize(
("test_secret_volumes", "expected_secret_volumes"),
(
(
[
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
[
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
),
),
)
def test_get_secret_volumes(
self, action_config, test_secret_volumes, expected_secret_volumes
):
action_config.config_dict["secret_volumes"] = test_secret_volumes
secret_volumes = action_config.get_secret_volumes()
assert secret_volumes == expected_secret_volumes

def test_get_executor_default(self, action_config):
assert action_config.get_executor() == "paasta"

Expand Down Expand Up @@ -787,6 +819,15 @@ def test_format_tron_action_dict_paasta(self):
"disk": 42,
"pool": "special_pool",
"env": {"SHELL": "/bin/bash"},
"secret_volumes": [
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
"extra_volumes": [
{"containerPath": "/nail/tmp", "hostPath": "/nail/tmp", "mode": "RW"}
],
Expand Down Expand Up @@ -835,6 +876,15 @@ def test_format_tron_action_dict_paasta(self):
"mem": 1200,
"disk": 42,
"env": mock.ANY,
"secret_volumes": [
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
"extra_volumes": [
{"container_path": "/nail/tmp", "host_path": "/nail/tmp", "mode": "RW"}
],
Expand Down Expand Up @@ -875,6 +925,15 @@ def test_format_tron_action_dict_spark(self):
"disk": 42,
"pool": "special_pool",
"env": {"SHELL": "/bin/bash"},
"secret_volumes": [
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
"extra_volumes": [
{"containerPath": "/nail/tmp", "hostPath": "/nail/tmp", "mode": "RW"}
],
Expand Down Expand Up @@ -1003,6 +1062,15 @@ def test_format_tron_action_dict_spark(self):
"mem": 1200,
"disk": 42,
"env": mock.ANY,
"secret_volumes": [
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
"extra_volumes": [
{"container_path": "/nail/tmp", "host_path": "/nail/tmp", "mode": "RW"}
],
Expand Down Expand Up @@ -1094,6 +1162,7 @@ def test_format_tron_action_dict_paasta_k8s_service_account(self):
"env": mock.ANY,
"secret_env": {},
"field_selector_env": {"PAASTA_POD_IP": {"field_path": "status.podIP"}},
"secret_volumes": [],
"extra_volumes": [],
"service_account_name": "a-magic-sa",
}
Expand Down Expand Up @@ -1140,6 +1209,15 @@ def test_format_tron_action_dict_paasta_k8s(
"disk": 42,
"pool": "special_pool",
"env": {"SHELL": "/bin/bash", "SOME_SECRET": "SECRET(secret_name)"},
"secret_volumes": [
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
"extra_volumes": [
{"containerPath": "/nail/tmp", "hostPath": "/nail/tmp", "mode": "RW"}
],
Expand Down Expand Up @@ -1218,6 +1296,15 @@ def test_format_tron_action_dict_paasta_k8s(
"key": "secret_name",
}
},
"secret_volumes": [
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
"field_selector_env": {"PAASTA_POD_IP": {"field_path": "status.podIP"}},
"extra_volumes": [
{"container_path": "/nail/tmp", "host_path": "/nail/tmp", "mode": "RW"}
Expand Down Expand Up @@ -1248,6 +1335,15 @@ def test_format_tron_action_dict_paasta_no_branch_dict(self):
"disk": 42,
"pool": "special_pool",
"env": {"SHELL": "/bin/bash"},
"secret_volumes": [
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
"extra_volumes": [
{"containerPath": "/nail/tmp", "hostPath": "/nail/tmp", "mode": "RW"}
],
Expand Down Expand Up @@ -1284,6 +1380,15 @@ def test_format_tron_action_dict_paasta_no_branch_dict(self):
"mem": 1200,
"disk": 42,
"env": mock.ANY,
"secret_volumes": [
{
"secret_volume_name": "abc",
"secret_name": "secret1",
"container_path": "/b/c",
"default_mode": "0644",
"items": [{"key": "secret1", "path": "abc"}],
}
],
"extra_volumes": [
{"container_path": "/nail/tmp", "host_path": "/nail/tmp", "mode": "RW"}
],
Expand Down Expand Up @@ -1430,7 +1535,7 @@ def test_create_complete_config_e2e(self, tmpdir):
# that are not static, this will cause continuous reconfiguration, which
# will add significant load to the Tron API, which happened in DAR-1461.
# but if this is intended, just change the hash.
assert hasher.hexdigest() == "f740410f7ae2794f9924121c1115e15d"
assert hasher.hexdigest() == "35972651618a848ac6bf7947245dbaea"

def test_override_default_pool_override(self, tmpdir):
soa_dir = tmpdir.mkdir("test_create_complete_config_soa")
Expand Down
18 changes: 18 additions & 0 deletions unique-run
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
apiVersion: v1
kind: Pod
metadata:
labels:
spark: exec-<MagicMock name='get_spark_app_name()' id='1405254751-rkvy
spec:
affinity:
podAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchExpressions:
- key: spark
operator: In
values:
- exec-<MagicMock name='get_spark_app_name()' id='1405254751-rkvy
topologyKey: topology.kubernetes.io/hostname
weight: 95

0 comments on commit e9ccd71

Please sign in to comment.