diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index a66efb4b117..20d26ca5106 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -3,6 +3,7 @@ dataproc { applicationName = "firecloud:leonardo" dataprocDockerImage = "gcr.io/broad-dsde-prod/leonardo-notebooks:prod" + defaultExecutionTimeout = 30 minutes jupyterServerName = "jupyter-server" firewallRuleName = "leonardo-notebooks-rule" networkTag = "leonardo" diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/Boot.scala b/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/Boot.scala index 2ef48bd426d..c6feddbbb1d 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/Boot.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/Boot.scala @@ -77,7 +77,7 @@ object Boot extends App with LazyLogging { } val (leoServiceAccountEmail, leoServiceAccountPemFile) = serviceAccountProvider.getLeoServiceAccountAndKey - val gdDAO = new HttpGoogleDataprocDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google", NetworkTag(dataprocConfig.networkTag), dataprocConfig.vpcNetwork.map(VPCNetworkName), dataprocConfig.vpcSubnet.map(VPCSubnetName), dataprocConfig.dataprocDefaultRegion) + val gdDAO = new HttpGoogleDataprocDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google", NetworkTag(dataprocConfig.networkTag), dataprocConfig.vpcNetwork.map(VPCNetworkName), dataprocConfig.vpcSubnet.map(VPCSubnetName), dataprocConfig.dataprocDefaultRegion, dataprocConfig.defaultExecutionTimeout) val googleComputeDAO = new HttpGoogleComputeDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google") val googleIamDAO = new HttpGoogleIamDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google") val googleStorageDAO = new HttpGoogleStorageDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google") diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/DataprocConfig.scala b/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/DataprocConfig.scala index 68e2d2abd16..09ee8380bf2 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/DataprocConfig.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/DataprocConfig.scala @@ -2,12 +2,15 @@ package org.broadinstitute.dsde.workbench.leonardo.config import org.broadinstitute.dsde.workbench.model.google.GoogleProject +import scala.concurrent.duration.FiniteDuration + case class DataprocConfig( applicationName: String, dataprocDefaultRegion: String, leoGoogleProject: GoogleProject, dataprocDockerImage: String, clusterUrlBase: String, + defaultExecutionTimeout: FiniteDuration, jupyterServerName: String, firewallRuleName: String, networkTag: String, diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/package.scala b/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/package.scala index 6ad7e1d8209..876606de615 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/package.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/package.scala @@ -23,6 +23,7 @@ package object config { GoogleProject(config.getString("leoGoogleProject")), config.getString("dataprocDockerImage"), config.getString("clusterUrlBase"), + toScalaDuration(config.getDuration("defaultExecutionTimeout")), config.getString("jupyterServerName"), config.getString("firewallRuleName"), config.getString("networkTag"), diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/google/HttpGoogleDataprocDAO.scala b/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/google/HttpGoogleDataprocDAO.scala index 723ed30b4bc..c56c06dee01 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/google/HttpGoogleDataprocDAO.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/google/HttpGoogleDataprocDAO.scala @@ -27,6 +27,7 @@ import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GcsPath, G import org.broadinstitute.dsde.workbench.model.{UserInfo, WorkbenchEmail, WorkbenchException, WorkbenchUserId} import scala.collection.JavaConverters._ +import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future} import scala.util.Try @@ -36,7 +37,8 @@ class HttpGoogleDataprocDAO(appName: String, networkTag: NetworkTag, vpcNetwork: Option[VPCNetworkName], vpcSubnet: Option[VPCSubnetName], - defaultRegion: String) + defaultRegion: String, + defaultExecutionTimeout: FiniteDuration) (implicit override val system: ActorSystem, override val executionContext: ExecutionContext) extends AbstractHttpGoogleDAO(appName, googleCredentialMode, workbenchMetricBaseName) with GoogleDataprocDAO { @@ -238,7 +240,7 @@ class HttpGoogleDataprocDAO(appName: String, // Create a NodeInitializationAction, which specifies the executable to run on a node. // This executable is our init-actions.sh, which will stand up our jupyter server and proxy. - val initActions = Seq(new NodeInitializationAction().setExecutableFile(initScript.toUri)) + val initActions = Seq(new NodeInitializationAction().setExecutableFile(initScript.toUri).setExecutionTimeout(finiteDurationToGoogleDuration(defaultExecutionTimeout))) // Create a config for the master node, if properties are not specified in request, use defaults val masterConfig = new InstanceGroupConfig() @@ -411,6 +413,11 @@ class HttpGoogleDataprocDAO(appName: String, } yield parseZone(zoneUri) } + //Note that this conversion will shave off anything smaller than a second in magnitude + private def finiteDurationToGoogleDuration(duration: FiniteDuration): String = { + s"${duration.toSeconds}s" + } + private implicit class GoogleExceptionSupport[A](future: Future[A]) { def handleGoogleException(project: GoogleProject, context: Option[String] = None): Future[A] = { future.recover { diff --git a/src/test/resources/reference.conf b/src/test/resources/reference.conf index c4ea0d1d8fc..6899fe6d4f2 100644 --- a/src/test/resources/reference.conf +++ b/src/test/resources/reference.conf @@ -35,6 +35,7 @@ dataproc { leoGoogleProject = "test-bucket" dataprocDockerImage = "testrepo/test" clusterUrlBase = "http://leonardo/" + defaultExecutionTimeout = 30 minutes jupyterServerName = "test-server" createClusterAsPetServiceAccount = false firewallRuleName = "test-rule"