From e0f8c596e3d738316bcb720b9ef3062bf9caa7cf Mon Sep 17 00:00:00 2001 From: Louis CAD Date: Mon, 31 Aug 2020 17:14:00 +0200 Subject: [PATCH] Improve publishing process Now, we upload without publishing, we check the uploaded artifacts, and then we attempt publishing uploaded artifacts. --- .github/workflows/release.yml | 45 ++++-- .../src/main/kotlin/publishing/Publishing.kt | 20 ++- gradle.properties | 5 + settings.gradle.kts | 6 +- tools/publication-checker/build.gradle.kts | 150 ++++++++++++++++++ .../src/androidMain/AndroidManifest.xml | 5 + versions.properties | 4 + 7 files changed, 216 insertions(+), 19 deletions(-) create mode 100644 tools/publication-checker/build.gradle.kts create mode 100644 tools/publication-checker/src/androidMain/AndroidManifest.xml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 00e50f989..acda54db8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,28 +8,51 @@ on: - master - develop jobs: - windows-publishing: + windows-upload: runs-on: windows-latest steps: - name: Checkout the repo uses: actions/checkout@v2 - - name: Publish Windows artifacts - run: ./gradlew publishMingwX64PublicationToBintrayRepository publishMingwX86PublicationToBintrayRepository - linux-publishing: + - name: Upload Windows artifacts + run: ./gradlew --scan publishMingwX64PublicationToBintrayRepository publishMingwX86PublicationToBintrayRepository + linux-upload: runs-on: ubuntu-latest steps: - name: Checkout the repo uses: actions/checkout@v2 - - name: Publish Linux artifacts - run: ./gradlew publishLinuxX64PublicationToBintrayRepository publishLinuxArm64PublicationToBintrayRepository publishLinuxArm32HfpPublicationToBintrayRepository - macos-publishing: + - name: Upload Linux artifacts + run: ./gradlew --scan publishLinuxArm64PublicationToBintrayRepository publishLinuxArm32HfpPublicationToBintrayRepository + macos-upload: runs-on: macOS-latest steps: - name: Checkout the repo uses: actions/checkout@v2 - - name: Publish all artifacts macOS can publish - run: ./gradlew publishAllPublicationsToBintrayRepository + - name: Upload all artifacts macOS can build + run: ./gradlew --scan publishAllPublicationsToBintrayRepository + windows-checking: + runs-on: windows-latest + needs: [windows-upload, linux-upload, macos-upload] + steps: + - name: Checkout the repo + uses: actions/checkout@v2 + - name: Check Windows build with latest artifacts + run: ./gradlew --scan -Psplitties.bintray.check=true :tools:publication-checker:build + macos-checking: + runs-on: macOS-latest + needs: [windows-upload, linux-upload, macos-upload] + steps: + - name: Checkout the repo + uses: actions/checkout@v2 + - name: Check macOS build with latest artifacts + run: ./gradlew --scan -Psplitties.bintray.check=true :tools:publication-checker:build + linux-checking-and-publish: + runs-on: ubuntu-latest + needs: [macos-checking, windows-checking] + steps: + - name: Checkout the repo + uses: actions/checkout@v2 + - name: Check Linux build with latest artifacts, then publish + run: ./gradlew --scan -Psplitties.bintray.check=true :tools:publication-checker:build publishBintrayRelease env: - GRADLE_OPTS: -Dorg.gradle.configureondemand=true -Dorg.gradle.parallel=true -Dkotlin.incremental=false -Dorg.gradle.project.kotlin.incremental.multiplatform=false -Dorg.gradle.project.kotlin.native.disableCompilerDaemon=true -Dorg.gradle.project.buildScan.termsOfServiceAgree=yes -Dorg.gradle.jvmargs="-Xmx3g -XX:MaxPermSize=2048m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8" - ORG_GRADLE_PROJECT_bintray_user: ${{ secrets.BINTRAY_USER }} + GRADLE_OPTS: -Dorg.gradle.configureondemand=true -Dorg.gradle.parallel=true -Dkotlin.incremental=false -Dorg.gradle.project.kotlin.incremental.multiplatform=false -Dorg.gradle.project.kotlin.native.disableCompilerDaemon=true -Dorg.gradle.jvmargs="-Xmx3g -XX:MaxPermSize=2048m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8" ORG_GRADLE_PROJECT_bintray_api_key: ${{ secrets.BINTRAY_API_KEY }} diff --git a/buildSrc/src/main/kotlin/publishing/Publishing.kt b/buildSrc/src/main/kotlin/publishing/Publishing.kt index a7c6bf94e..5d86f94b4 100644 --- a/buildSrc/src/main/kotlin/publishing/Publishing.kt +++ b/buildSrc/src/main/kotlin/publishing/Publishing.kt @@ -57,21 +57,27 @@ fun PublishingExtension.setupAllPublications(project: Project) { private fun PublishingExtension.setupPublishRepo(project: Project) { repositories { + + fun bintrayProperty(keySuffix: String): String = project.property("splitties.bintray.$keySuffix") as String + + val bintrayUsername = bintrayProperty("user") + val bintrayApiKey = project.findProperty("bintray_api_key") as String? ?: return@repositories maven { - val isDevVersion = project.isDevVersion name = "bintray" - val bintrayUsername = "louiscad" - val bintrayRepoName = if (isDevVersion) "splitties-dev" else "maven" - val bintrayPackageName = "splitties" + val isDevVersion = project.isDevVersion + val bintrayRepoName = bintrayProperty(if (isDevVersion) "repo.dev" else "repo.release") + val bintrayPackageName = bintrayProperty("package") setUrl( "https://api.bintray.com/maven/" + "$bintrayUsername/$bintrayRepoName/$bintrayPackageName/;" + - "publish=1;" + // Might conflict with override. TODO: Revert or remove this comment based on results "override=1" + // We don't (no longer) publish on upload because it increases the risk of the bintray API returning + // HTTP 405 or 409 errors, it allows publishing an invalid release. + // We publish later once we validated all artifacts. ) credentials { - username = project.findProperty("bintray_user") as String? - password = project.findProperty("bintray_api_key") as String? + username = bintrayUsername + password = bintrayApiKey } } } diff --git a/gradle.properties b/gradle.properties index ca7bcf61f..ac9e21d2a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -31,3 +31,8 @@ android.nonTransitiveRClass=true # Required to publish to Nexus (see https://github.com/gradle/gradle/issues/11308) systemProp.org.gradle.internal.publish.checksums.insecure=true + +splitties.bintray.repo.dev=splitties-dev +splitties.bintray.repo.release=maven +splitties.bintray.package=splitties +splitties.bintray.user=louiscad diff --git a/settings.gradle.kts b/settings.gradle.kts index a632f7bfa..515faea24 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -30,7 +30,7 @@ plugins { gradleEnterprise { buildScan { termsOfServiceUrl = "https://gradle.com/terms-of-service" - termsOfServiceAgree = settings.extra.properties["buildScan.termsOfServiceAgree"] as String? + termsOfServiceAgree = "yes" } } @@ -103,3 +103,7 @@ arrayOf( ).forEach { include(":samples:$it") } include("test-helpers") + +if (extra.properties["splitties.bintray.check"].toString().toBoolean()) { + include(":tools:publication-checker") +} diff --git a/tools/publication-checker/build.gradle.kts b/tools/publication-checker/build.gradle.kts new file mode 100644 index 000000000..21ca0a3f2 --- /dev/null +++ b/tools/publication-checker/build.gradle.kts @@ -0,0 +1,150 @@ +/* + * Copyright 2020 Louis Cognault Ayeva Derman. Use of this source code is governed by the Apache 2.0 license. + */ + +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import retrofit2.Response +import kotlin.time.* + +buildscript { + repositories { mavenCentral() } + dependencies.classpath(Square.okHttp3.okHttp) + dependencies.classpath(Square.retrofit2.retrofit) +} + +plugins { + id("com.android.library") + kotlin("multiplatform") +} + +fun bintrayProperty(keySuffix: String): String = project.property("splitties.bintray.$keySuffix") as String + +val bintrayUsername = bintrayProperty("user") +val bintrayApiKey = project.property("bintray_api_key") as String +val isDevVersion = project.isDevVersion +val bintrayRepoName = bintrayProperty(if (isDevVersion) "repo.dev" else "repo.release") +val bintrayPackageName = bintrayProperty("package") + +repositories { + maven("https://dl.bintray.com/$bintrayUsername/$bintrayRepoName/") { + name = "bintray/$bintrayUsername/$bintrayRepoName" + credentials { + username = bintrayUsername + password = bintrayApiKey + } + } +} + +tasks.matching { it.name.startsWith("lint") }.configureEach { enabled = false } + +android { + setDefaults() +} + +kotlin { + android() + js { useCommonJs() } + macos() + ios(supportArm32 = true) + watchos() + tvos() + mingw(x64 = true, x86 = true) + linux(x64 = true, arm32Hfp = true, arm64 = true, mips32 = true, mipsel32 = true) + sourceSets { + commonMain.dependencies { + + val group = "com.louiscad.splitties" + val version = thisLibraryVersion + + rootProject.rootDir.resolve("modules").listFiles { file -> + file.isDirectory && file.resolve("build.gradle.kts").exists() + }!!.also { check(it.isNotEmpty()) }.forEach { + implementation("$group:splitties-${it.name}") { + version { strictly(version) } + } + } + + rootProject.rootDir.resolve("fun-packs").listFiles { file -> + file.isDirectory && file.resolve("build.gradle.kts").exists() + }!!.also { check(it.isNotEmpty()) }.forEach { + implementation("$group:splitties-fun-pack-${it.name}") { + version { strictly(version) } + } + } + } + } +} + +rootProject.tasks.register("publishBintrayRelease") { + group = "publishing" + description = "Publishes the artifacts uploaded on bintray for this version" + + dependsOn(tasks.named("build")) // Ensure publish happens after concurrent build check. + + @Suppress("experimental_is_not_enabled") + @OptIn(ExperimentalTime::class) + doFirst { + + // These durations might need to be raised for larger projects... + // ...(but at this point, isn't the project too large?) + val requestReadTimeout = 10.minutes + val retryBackOff = 30.seconds + val giveUpAfter = 1.hours + + val subject = bintrayUsername + val repo = bintrayRepoName + val version = thisLibraryVersion + val `package` = bintrayPackageName + + val request = okhttp3.Request.Builder() + .header( + name = "Authorization", + value = okhttp3.Credentials.basic(username = bintrayUsername, password = bintrayApiKey) + ) + // Bintray API reference: https://bintray.com/docs/api/#_publish_discard_uploaded_content + .url("https://bintray.com/api/v1/content/$subject/$repo/$`package`/$version/publish") + .post("""{"publish_wait_for_secs":-1}""".toRequestBody("application/json".toMediaType())) + .build() + val httpClient = okhttp3.OkHttpClient.Builder() + .readTimeout(requestReadTimeout.toJavaDuration()) + .build() + + /** + * If [isLastAttempt] is true, any failure will be thrown as an exception. + */ + fun attemptPublishing(isLastAttempt: Boolean = false): Boolean { + println("Attempting bintray publish") + try { + httpClient.newCall(request).execute().use { response -> + if (response.isSuccessful) { + println(response.body?.string()) + return true + } + if (isLastAttempt.not()) when (val code = response.code) { + 408, 405 -> { + logger.error("Publish attempt failed (http $code)") + logger.info(response.body?.string() ?: "") + return false + } + } + throw retrofit2.HttpException(Response.error(response.code, response.body!!)) + } + } catch (e: java.io.IOException) { + if (isLastAttempt) throw e + logger.error("Publish attempt failed with ${e.javaClass.simpleName}: ${e.message}") + return false + } + } + + println("Will attempt bintray publishing on ${request.url}") + + val deadline = TimeSource.Monotonic.markNow() + giveUpAfter + do { + val didSucceed = attemptPublishing(isLastAttempt = deadline.hasPassedNow()) + if (didSucceed.not()) Thread.sleep(retryBackOff.toLongMilliseconds()) + } while (didSucceed.not()) + + println("Bintray publishing successful!") + } +} diff --git a/tools/publication-checker/src/androidMain/AndroidManifest.xml b/tools/publication-checker/src/androidMain/AndroidManifest.xml new file mode 100644 index 000000000..9ff8c019e --- /dev/null +++ b/tools/publication-checker/src/androidMain/AndroidManifest.xml @@ -0,0 +1,5 @@ + + + diff --git a/versions.properties b/versions.properties index 3db63364a..81fd26b22 100644 --- a/versions.properties +++ b/versions.properties @@ -154,3 +154,7 @@ version.kotlinx.coroutines=1.3.8 version.robolectric=4.4 ## # available=4.5-SNAPSHOT + +version.okhttp3=4.8.1 + +version.retrofit2=2.9.0