Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit a009eba

Browse files
author
Dan King
committedNov 10, 2023
[query] gradlebuildcache
1 parent 96713a9 commit a009eba

File tree

4 files changed

+93
-39
lines changed

4 files changed

+93
-39
lines changed
 

‎build.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -742,6 +742,9 @@ steps:
742742
cpu: '2'
743743
script: |
744744
set -ex
745+
746+
export GRADLE_CACHE_URI={{ global.test_storage_uri }}/gradlebuildcache
747+
745748
cd /io/repo/hail
746749
chmod 755 ./gradlew
747750
time retry ./gradlew --version
@@ -775,6 +778,9 @@ steps:
775778
cpu: '2'
776779
script: |
777780
set -ex
781+
782+
export GRADLE_CACHE_URI={{ global.test_storage_uri }}/gradlebuildcache
783+
778784
cd /io/repo/hail
779785
chmod 755 ./gradlew
780786
time retry ./gradlew --version
@@ -801,6 +807,9 @@ steps:
801807
cpu: '2'
802808
script: |
803809
set -ex
810+
811+
export GRADLE_CACHE_URI={{ global.test_storage_uri }}/gradlebuildcache
812+
804813
cd /io/repo/hail
805814
chmod 755 ./gradlew
806815
time retry ./gradlew --version
@@ -837,6 +846,9 @@ steps:
837846
cpu: '2'
838847
script: |
839848
set -ex
849+
850+
export GRADLE_CACHE_URI={{ global.test_storage_uri }}/gradlebuildcache
851+
840852
cd /io/repo/hail
841853
chmod 755 ./gradlew
842854
time retry ./gradlew --version
@@ -3236,6 +3248,8 @@ steps:
32363248
script: |
32373249
set -ex
32383250
3251+
export GRADLE_CACHE_URI={{ global.test_storage_uri }}/gradlebuildcache
3252+
32393253
cd /io/repo
32403254
32413255
gcloud auth activate-service-account --key-file=/test-dataproc-service-account-key/test-dataproc-service-account-key.json
@@ -3278,6 +3292,8 @@ steps:
32783292
script: |
32793293
set -ex
32803294
3295+
export GRADLE_CACHE_URI={{ global.test_storage_uri }}/gradlebuildcache
3296+
32813297
cd /io/repo
32823298
32833299
gcloud auth activate-service-account --key-file=/test-dataproc-service-account-key/test-dataproc-service-account-key.json
@@ -3321,6 +3337,8 @@ steps:
33213337
set -ex
33223338
cd /io
33233339
3340+
export GRADLE_CACHE_URI={{ global.test_storage_uri }}/gradlebuildcache
3341+
33243342
gcloud auth activate-service-account --key-file=/ci-deploy-0-1--hail-is-hail/ci-deploy-0-1--hail-is-hail.json
33253343
33263344
gcloud auth -q configure-docker {{ global.docker_prefix.split('/')[0] }}

‎gcsbuildcache/build.gradle

+12-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
1-
plugins {
2-
id 'java'
3-
id 'scala'
1+
buildscript {
2+
repositories {
3+
mavenCentral()
4+
}
45
}
56

67
plugins {
8+
id 'java'
9+
id 'scala'
710
id "com.gradle.plugin-publish" version "1.0.0"
11+
id 'com.github.johnrengelman.shadow' version '8.1.1'
812
}
913

1014
group = "is.hail.gcsbuildcache"
@@ -33,3 +37,8 @@ gradlePlugin {
3337
}
3438
}
3539
}
40+
41+
shadowJar {
42+
archiveClassifier = ''
43+
}
44+

‎gcsbuildcache/src/main/scala/is/hail/gcsbuildcache/GCSBuildCache.scala

+54-29
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,28 @@ import org.gradle.caching.configuration.AbstractBuildCache
77
import org.gradle.caching.{BuildCacheEntryWriter, BuildCacheEntryReader, BuildCacheService, BuildCacheServiceFactory, BuildCacheKey}
88
import com.google.auth.oauth2.GoogleCredentials
99
import com.google.cloud.storage.{Bucket, StorageException, StorageOptions, BlobId, BlobInfo}
10+
import com.google.cloud.storage.Storage.{BlobWriteOption, BlobTargetOption}
1011
import com.google.common.io.FileBackedOutputStream
1112
import java.nio.channels.Channels
1213
import java.time.OffsetDateTime
1314
import java.time.temporal.ChronoUnit
1415

1516

16-
class GCSBuildCacheConfiguration(
17-
var gcsURL: String = "",
18-
) extends AbstractBuildCache
17+
// Gradle requirs a no-argument constructor. A one-argument Scala constructor with default arguments
18+
// *does not work*.
19+
class GCSBuildCacheConfiguration() extends AbstractBuildCache {
20+
// Gradle requires literal setters and getters. Do not try to expose a mutable field directly to
21+
// gradle, you will get errors like "Could not set unknown property 'gcsURL'"
22+
private[this] var _gcsURL: String = null
23+
def setGcsURL(gcsURL: String) = _gcsURL = gcsURL
24+
def getGcsURL: String = _gcsURL
25+
def getIsPush: Boolean = isPush()
26+
def setIsPush(isPush: Boolean) = setPush(isPush)
27+
// Gradle cannot directly modify isEnabled/setEnabled because they do not follow the get/set
28+
// pattern.
29+
def getEnabled: Boolean = isEnabled()
30+
override def setEnabled(enabled: Boolean) = super.setEnabled(enabled)
31+
}
1932

2033
class GCSBuildCache(
2134
conf: GCSBuildCacheConfiguration
@@ -27,47 +40,59 @@ class GCSBuildCache(
2740
.getService
2841

2942
override def store(key: BuildCacheKey, writer: BuildCacheEntryWriter): Unit = {
30-
val path = conf.gcsURL + "/" + key.hashCode.toString
31-
val blobinfo = BlobInfo.newBuilder(BlobId.fromGsUtilUri(path)).build()
32-
val value = new FileBackedOutputStream(8 * 1024 * 1024, true)
33-
writer.writeTo(value)
34-
val is = value.asByteSource().openBufferedStream()
3543
try {
36-
storage.create(blobinfo, is)
37-
System.out.println(s"cache stored $key")
44+
val path = conf.getGcsURL + "/" + key.hashCode
45+
val blobinfo = BlobInfo.newBuilder(BlobId.fromGsUtilUri(path)).build()
46+
val value = new FileBackedOutputStream(8 * 1024 * 1024, true)
47+
writer.writeTo(value)
48+
log.lifecycle(s"storing cache $key ${key.hashCode} $path")
49+
val is = value.asByteSource().openBufferedStream()
50+
try {
51+
val onlyCreateIfNotExists = BlobWriteOption.doesNotExist()
52+
storage.create(blobinfo, is, onlyCreateIfNotExists)
53+
log.lifecycle(s"cache stored $key ${key.hashCode} $path")
54+
} catch {
55+
case exc: StorageException =>
56+
throw new RuntimeException(s"$key ${key.hashCode} could not be stored in cache at $path", exc)
57+
} finally {
58+
is.close()
59+
}
3860
} catch {
39-
case exc: StorageException =>
40-
throw new RuntimeException(s"$key could not be stored in cache at $path", exc)
41-
} finally {
42-
is.close()
61+
case exc: Exception =>
62+
// Gradle will silence all exceptions and quietly disable the build cache, so we loudly
63+
// print the exception.
64+
exc.printStackTrace()
65+
throw exc
4366
}
4467
}
4568

4669
override def load(key: BuildCacheKey, reader: BuildCacheEntryReader): Boolean = {
47-
System.out.println(s"checking cache key $key")
48-
val path = conf.gcsURL + "/" + key.hashCode.toString
70+
val path = conf.getGcsURL + "/" + key.hashCode
4971
val blobid = BlobId.fromGsUtilUri(path)
72+
log.lifecycle(s"checking cache key $key ${key.hashCode} $path")
5073
try {
5174
val blob = storage.get(blobid)
75+
if (blob == null) {
76+
return false
77+
}
5278
reader.readFrom(Channels.newInputStream(blob.reader()))
53-
System.out.println(s"cache hit $key")
79+
log.lifecycle(s"cache hit $key ${key.hashCode} $path")
5480

5581
if (blob.getCreateTimeOffsetDateTime().until(OffsetDateTime.now(), ChronoUnit.SECONDS) > 24 * 60 * 60) {
5682
val blobinfo = BlobInfo.newBuilder(blobid).build()
57-
System.out.println(s"will refreshing cache $key")
58-
storage.create(blobinfo, blob.getContent())
59-
System.out.println(s"refreshed cache $key")
83+
log.lifecycle(s"will refreshing cache $key ${key.hashCode} $path")
84+
val onlyUpdateIfNoOneHasBeatenMeToIt = BlobTargetOption.generationMatch(blob.getGeneration)
85+
storage.create(blobinfo, blob.getContent(), onlyUpdateIfNoOneHasBeatenMeToIt)
86+
log.lifecycle(s"refreshed cache $key ${key.hashCode} $path")
6087
}
6188
return true
6289
} catch {
63-
case exc: StorageException if exc.getCode == 404 =>
64-
System.out.println(s"cache miss $key")
65-
return false
66-
case exc: StorageException =>
67-
throw new RuntimeException(s"$key could not be loaded from cache at $path", exc)
90+
case exc: Exception =>
91+
// Gradle will silence all exceptions and quietly disable the build cache, so we loudly
92+
// print the exception.
93+
exc.printStackTrace()
94+
throw exc
6895
}
69-
70-
return false
7196
}
7297

7398
override def close(): Unit = {}
@@ -78,13 +103,13 @@ class GCSBuildCacheServiceFactory extends BuildCacheServiceFactory[GCSBuildCache
78103
conf: GCSBuildCacheConfiguration,
79104
describer: BuildCacheServiceFactory.Describer
80105
): BuildCacheService = {
81-
if (conf.gcsURL == null || conf.gcsURL == "") {
106+
if (conf.getGcsURL == null || conf.getGcsURL == "") {
82107
throw new GradleException("gcsURL must be set.")
83108
}
84109

85110
describer
86111
.`type`("Google Cloud Storage")
87-
.config("gcsURL", conf.gcsURL)
112+
.config("gcsURL", conf.getGcsURL)
88113

89114
return new GCSBuildCache(conf)
90115
}

‎hail/settings.gradle

+9-7
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
buildscript {
2-
dependencies {
3-
classpath files("../gcsbuildcache/build/libs/gcsbuildcache-0.0.1.jar")
4-
}
1+
pluginManagement {
2+
includeBuild '../gcsbuildcache'
3+
}
4+
plugins {
5+
id "is.hail.gcsbuildcache"
56
}
6-
77
apply plugin: 'is.hail.gcsbuildcache'
88

99
buildCache {
@@ -14,8 +14,10 @@ buildCache {
1414
enabled = false
1515
}
1616

17-
remote(is.hail.buildcache.GCSBuildCacheConfiguration) {
18-
gcsURL = 'gs://danking/gradlebuildcache' // System.env.TEST_STORAGE_URI
17+
remote(is.hail.gcsbuildcache.GCSBuildCacheConfiguration) {
18+
gcsURL = System.env.GRADLE_CACHE_URI
19+
isPush = true
20+
enabled = System.env.GRADLE_CACHE_URI != null
1921
}
2022
}
2123

0 commit comments

Comments
 (0)
Please sign in to comment.