Improve DSEngine workload deployment script and options.

scale-tests/configs/jupiter-options.json

@@ -1,17 +1,17 @@
 {
   "service": {
-    "name": "jupyter/jupyter",
-    "service_account": "jupyter__jupyter",
-    "service_account_secret": "jupyter__jupyter-secret",
+    "name": "data-services/jupyter",
+    "service_account": "data_services__jupyter",
+    "service_account_secret": "data_services__jupyter-secret",
     "gpu": {
       "enabled": true
     },
     "virtual_network_enabled": true,
     "virtual_network_name": "dcos"
   },
   "spark": {
-    "spark_mesos_role": "jupyter__jupyter",
-    "spark_mesos_principal": "jupyter__jupyter",
-    "spark_mesos_secret": "jupyter__jupyter-secret"
+    "spark_mesos_role": "data_services__jupyter",
+    "spark_mesos_principal": "data_services__jupyter",
+    "spark_mesos_secret": "data_services__jupyter-secret"
   }
-}
+}

scale-tests/deploy_dsengine.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+set -x
+
+create_service_account() {
+  SERVICE_ACCOUNT="${1}"
+  SECRET_NAME="${SERVICE_ACCOUNT}-secret"
+
+  dcos security org service-accounts delete "${SERVICE_ACCOUNT}"
+  dcos security secrets delete "${SECRET_NAME}"
+
+  dcos security org service-accounts keypair private.pem public.pem
+  dcos security org service-accounts create \
+       -p public.pem \
+       -d "Service account for ${SERVICE_ACCOUNT}" "${SERVICE_ACCOUNT}"
+  dcos security secrets create-sa-secret \
+       --strict private.pem \
+       "${SERVICE_ACCOUNT}" \
+       "${SECRET_NAME}"
+
+  rm -f private.pem public.pem
+}
+
+grant_permissions() {
+  SERVICE_ACCOUNT="${1}"
+
+  echo "Granting permissions to Service Account ${SERVICE_ACCOUNT}"
+
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:master:task:user:nobody" \
+       create
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:agent:task:user:nobody" \
+       create
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:secrets:list:default:__dcos_base64__hdfs_jupyter_keytab" \
+       read
+}
+
+grant_spark_permissions() {
+  SERVICE_ACCOUNT=$1
+  echo "Granting Spark permissions to Jupyter Service Account ${SERVICE_ACCOUNT}"
+
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:master:framework:role:${SERVICE_ACCOUNT}" \
+       create
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:master:reservation:role:${SERVICE_ACCOUNT}" \
+       create
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:master:reservation:principal:${SERVICE_ACCOUNT}" \
+       delete
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:master:volume:role:${SERVICE_ACCOUNT}" \
+       create
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:master:volume:principal:${SERVICE_ACCOUNT}" \
+       delete
+
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:master:task:role:${SERVICE_ACCOUNT}" \
+       create
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:master:task:principal:${SERVICE_ACCOUNT}" \
+       create
+  dcos security org users grant "${SERVICE_ACCOUNT}" \
+       "dcos:mesos:master:task:app_id:data-services/jupyter" \
+       create
+}
+create_service_account data_services__jupyter
+
+grant_permissions data_services__jupyter
+grant_spark_permissions data_services__jupyter
+
+dcos package install --yes data-science-engine \
+     --options=scale-tests/configs/jupiter-options.json
+
+# Run the following in the Jupyter notebook UI (password: jupyter):
+#
+# ! spark-submit \
+  #     --conf spark.mesos.gpus.max=40 \
+  #     --conf spark.cores.max=40 \
+  #     --conf spark.mesos.executor.gpus=1 \
+  #     --conf spark.executor.cores=1 \
+  #     --verbose \
+  #     --class MockTaskRunner \
+  #     https://infinity-artifacts.s3.amazonaws.com/scale-tests/dcos-spark-scala-tests-assembly-2.4.0-20190325.jar 5000 10