diff --git a/.github/actions/azure-login/action.yml b/.github/actions/azure-login/action.yml index e92942e40..ea4a44d5a 100644 --- a/.github/actions/azure-login/action.yml +++ b/.github/actions/azure-login/action.yml @@ -27,7 +27,7 @@ runs: subscription-id: ${{ inputs.AZURE_SUBSCRIPTION_ID }} - name: Login to Azure Container Registry - uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 with: registry: adoptopenjdkacr.azurecr.io username: ${{ inputs.AZURE_CLIENT_ID }} diff --git a/.github/actions/do-login/action.yml b/.github/actions/do-login/action.yml index 9670c59e5..7b28fdfe9 100644 --- a/.github/actions/do-login/action.yml +++ b/.github/actions/do-login/action.yml @@ -8,7 +8,7 @@ runs: using: composite steps: - name: Login to DigitalOcean - uses: digitalocean/action-doctl@cf10ddb4467f6fb84c01f7cc5f099cec71eabc93 # v2.4.1 + uses: digitalocean/action-doctl@e5cb5b0cde9789f79c5115c2c4d902f38a708804 # v2.5.0 with: token: ${{ inputs.DIGITALOCEAN_ACCESS_TOKEN }} diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index d17ed5eba..84596fe68 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -4,6 +4,9 @@ inputs: DOCKER_REPO: description: 'Docker repository' required: true + DOCKER_FILE: + description: 'Dockerfile path' + required: true BUILD_ARGS: description: 'Build arguments' required: false @@ -11,9 +14,9 @@ runs: using: composite steps: - name: Build container image and push to Azure - uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4.2.1 + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 with: - file: ./deploy/Dockerfile + file: ${{ inputs.DOCKER_FILE }} tags: ${{ inputs.DOCKER_REPO }}:latest build-args: ${{ inputs.BUILD_ARGS }} push: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e90560fd7..512e01262 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,6 +4,20 @@ updates: directory: "/" schedule: interval: "daily" + # Maintain dependencies for GitHub composite Actions (/.github/actions) + # Waiting for supporting wildcards see https://github.com/dependabot/dependabot-core/issues/5137 + - package-ecosystem: "github-actions" + directory: "/.github/actions/azure-login" + schedule: + interval: "daily" + - package-ecosystem: "github-actions" + directory: "/.github/actions/do-login" + schedule: + interval: "daily" + - package-ecosystem: "github-actions" + directory: "/.github/actions/docker-build" + schedule: + interval: "daily" - package-ecosystem: "maven" directory: "/" schedule: @@ -12,3 +26,7 @@ updates: directory: "/" schedule: interval: "daily" + - package-ecosystem: docker + directory: "/deploy" + schedule: + interval: "daily" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5909ecc7..048ffff3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,13 +30,13 @@ jobs: adoptopenjdk) echo "ARGS=-Padoptopenjdk,-adoptium" >> $GITHUB_ENV ;; esac - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: persist-credentials: false - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 with: - java-version: '17' + java-version: '21' distribution: 'temurin' cache: 'maven' diff --git a/.github/workflows/deploy-adoptium.yml b/.github/workflows/deploy-adoptium.yml index 7a3a2f96d..be7251fbc 100644 --- a/.github/workflows/deploy-adoptium.yml +++ b/.github/workflows/deploy-adoptium.yml @@ -14,7 +14,7 @@ jobs: if: startsWith(github.repository, 'adoptium/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Login to Azure uses: ./.github/actions/azure-login @@ -36,6 +36,7 @@ jobs: uses: ./.github/actions/docker-build with: DOCKER_REPO: adoptopenjdkacr.azurecr.io/adoptopenjdk-${{ env.NAMESPACE }} + DOCKER_FILE: ./deploy/Dockerfile - name: Redeploy Kubernetes Updater and Frontend uses: ./.github/actions/kubectl-redeploy diff --git a/.github/workflows/deploy-adoptopenjdk.yml b/.github/workflows/deploy-adoptopenjdk.yml index 7c855bd35..3249a3d69 100644 --- a/.github/workflows/deploy-adoptopenjdk.yml +++ b/.github/workflows/deploy-adoptopenjdk.yml @@ -14,7 +14,7 @@ jobs: if: startsWith(github.repository, 'adoptium/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Login to Azure uses: ./.github/actions/azure-login @@ -36,6 +36,7 @@ jobs: uses: ./.github/actions/docker-build with: DOCKER_REPO: adoptopenjdkacr.azurecr.io/azure-${{ env.NAMESPACE }} + DOCKER_FILE: ./deploy/Dockerfile BUILD_ARGS: MAVEN_FLAGS="-Padoptopenjdk,-adoptium" - name: Redeploy Kubernetes Updater and Frontend diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d3d8a7b41..a05b176dc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,11 +14,11 @@ jobs: contents: write # for Git to git push steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 with: - java-version: '17' + java-version: '21' distribution: 'temurin' cache: 'maven' diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index a36fc474e..c51ee6e36 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -14,5 +14,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.2/apache-maven-3.9.2-bin.zip +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.5/apache-maven-3.9.5-bin.zip wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/.s2i/bin/assemble b/.s2i/bin/assemble deleted file mode 100644 index 25f5e3e21..000000000 --- a/.s2i/bin/assemble +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# BUILDS FOR ADOPTOPENJDK, NOT ADOPTIUM - -set -euxo pipefail - -if [ "$(ls /tmp/artifacts/ 2>/dev/null)" ]; then - mv /tmp/artifacts/.m2 ${HOME}/ -fi - -cd /tmp/src -./mvnw -B clean install -Padoptopenjdk,-adoptium - -if [ -v SWAGGERHUB_APIKEY ]; then - cd adoptium-frontend-parent/adoptium-api-v3-frontend - unzip target/quarkus-app/quarkus/generated-bytecode.jar META-INF/quarkus-generated-openapi-doc.JSON || true - ../../mvnw io.swagger:swaggerhub-maven-plugin:upload || true - cd ../.. -fi - -mkdir -p /deployments - -cp -r adoptium-frontend-parent/adoptium-api-v3-frontend/target/quarkus-app/* /deployments/ - -cp adoptium-updater-parent/adoptium-api-v3-updater/target/adoptium-api-v3-updater-*-jar-with-dependencies.jar /deployments/adoptium-api-v3-updater-runner.jar -mv /deployments/quarkus-run.jar /deployments/adoptium-api-v3-frontend.jar - -mkdir -p /deployments/scripts -chmod +x scripts/* -cp scripts/* /deployments/scripts/ - -rm -rf /tmp/jboss || true -rm -rf /tmp/hsperfdata_* || true -rm -rf /tmp/vert* || true -rm -rf /logs/ || true -rm /tmp/updater.log || true -rm /tmp/frontend.log || true diff --git a/.s2i/bin/save-artifacts b/.s2i/bin/save-artifacts deleted file mode 100644 index 83f41cea1..000000000 --- a/.s2i/bin/save-artifacts +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -pushd ${HOME} >/dev/null - -tar cf - ./.m2 - -popd >/dev/null \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a5a966516..4f66ec001 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,6 +34,24 @@ If you want to use the updater tool to add entries into the database, you need t The production server uses mongodb to store data, however you can also use Fongo. If you would like to install mongodb and are on mac, I used this [guide](https://zellwk.com/blog/install-mongodb/) which utilizes homebrew. You can also install `mongo` which is a command-line tool that gives you access to your mongodb, allowing you to manually search through the database. +### GitHub App Authentication + +The updater can be used with a GitHub Token or GitHub App. To use a GitHub app you need to generate an app on GitHub. Once you've done that you need to convert the key to PKCS#8 format using the following command: + +```bash +openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in your-rsa-private-key.pem -out pkcs8-key.pem +``` + +Once this is done you can export the following variables at runtime: + +```bash +export GITHUB_APP_ID="1234" +export GITHUB_APP_INSTALLATION_ID="1234" +export GITHUB_APP_PRIVATE_KEY=$'-----BEGIN PRIVATE KEY----- + +-----END PRIVATE KEY-----' +``` + ### Build Tool [Maven](https://maven.apache.org/index.html) is used to build the project. diff --git a/Dockerfile b/Dockerfile index 7f5d151e0..b0cbad58c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM eclipse-temurin:17 as build +FROM eclipse-temurin:21 as build RUN mkdir /tmp/build @@ -8,7 +8,7 @@ COPY . /tmp/build RUN ./mvnw clean install -Padoptium -FROM eclipse-temurin:17 +FROM eclipse-temurin:21 RUN mkdir -p /deployments diff --git a/adoptium-api-versions/pom.xml b/adoptium-api-versions/pom.xml index fb254ffa6..543329fde 100644 --- a/adoptium-api-versions/pom.xml +++ b/adoptium-api-versions/pom.xml @@ -9,18 +9,18 @@ 1.7.3 - 2.15.2 + 2.15.3 17 - 4.10.0 + 4.11.0 17 - 1.9.10 + 1.9.20 1.4.11 17 - 3.9.2 + 3.9.5 3.4.5 UTF-8 UTF-8 - 3.4.2 + 3.5.1 5.3.2 @@ -200,7 +200,7 @@ de.flapdoodle.embed de.flapdoodle.embed.mongo - 4.9.2 + 4.10.1 test @@ -247,7 +247,7 @@ org.jooq jooq - 3.18.6 + 3.18.7 org.slf4j @@ -263,22 +263,22 @@ com.microsoft.azure applicationinsights-runtime-attach - 3.4.17 + 3.4.18 io.opentelemetry.javaagent.instrumentation opentelemetry-javaagent-kotlinx-coroutines - 1.30.0-alpha + 1.31.0-alpha io.opentelemetry.javaagent.instrumentation opentelemetry-javaagent-netty-4.1 - 1.30.0-alpha + 1.31.0-alpha org.jboss.weld weld-junit5 - 4.0.0.Final + 4.0.1.Final test @@ -337,25 +337,31 @@ org.junit.jupiter junit-jupiter - 5.10.0 + 5.10.1 test io.smallrye smallrye-open-api-core - 3.6.2 + 3.7.0 + test + + + io.smallrye + smallrye-open-api-jaxrs + 3.7.0 test org.eclipse.jetty jetty-server - 12.0.1 + 12.0.3 test org.eclipse.jetty jetty-client - 12.0.1 + 12.0.3 org.bouncycastle @@ -405,7 +411,7 @@ com.expediagroup graphql-kotlin-ktor-client - 7.0.1 + 7.0.2 org.jetbrains.kotlinx @@ -416,7 +422,7 @@ com.expediagroup graphql-kotlin-client-jackson - 7.0.1 + 7.0.2 diff --git a/adoptium-frontend-parent/adoptium-api-v3-frontend/src/main/kotlin/net/adoptium/api/v3/routes/info/ReleaseListResource.kt b/adoptium-frontend-parent/adoptium-api-v3-frontend/src/main/kotlin/net/adoptium/api/v3/routes/info/ReleaseListResource.kt index f1426036b..70a273a66 100644 --- a/adoptium-frontend-parent/adoptium-api-v3-frontend/src/main/kotlin/net/adoptium/api/v3/routes/info/ReleaseListResource.kt +++ b/adoptium-frontend-parent/adoptium-api-v3-frontend/src/main/kotlin/net/adoptium/api/v3/routes/info/ReleaseListResource.kt @@ -151,6 +151,7 @@ constructor( semver ) .map { it.release_name } + .distinct() val pagedReleases = getPage(pageSize, page, releases, showPageCount ?: false) diff --git a/adoptium-frontend-parent/adoptium-api-v3-frontend/src/main/kotlin/net/adoptium/api/v3/routes/stats/DownloadStatsResource.kt b/adoptium-frontend-parent/adoptium-api-v3-frontend/src/main/kotlin/net/adoptium/api/v3/routes/stats/DownloadStatsResource.kt index be6586170..c48d545b8 100644 --- a/adoptium-frontend-parent/adoptium-api-v3-frontend/src/main/kotlin/net/adoptium/api/v3/routes/stats/DownloadStatsResource.kt +++ b/adoptium-frontend-parent/adoptium-api-v3-frontend/src/main/kotlin/net/adoptium/api/v3/routes/stats/DownloadStatsResource.kt @@ -21,6 +21,7 @@ import java.util.concurrent.CompletionStage import jakarta.enterprise.context.ApplicationScoped import jakarta.inject.Inject import jakarta.ws.rs.BadRequestException +import jakarta.ws.rs.DefaultValue import jakarta.ws.rs.GET import jakarta.ws.rs.Path import jakarta.ws.rs.PathParam @@ -63,17 +64,21 @@ class DownloadStatsResource { @GET @Schema(hidden = true) @Path("/total/{feature_version}") - @Operation(summary = "Get download stats for feature verson", description = "stats", hidden = true) + @Operation(summary = "Get download stats for feature version", description = "stats", hidden = true) fun getTotalDownloadStats( @Parameter(name = "feature_version", description = "Feature version (i.e 8, 9, 10...)", required = true) @PathParam("feature_version") - featureVersion: Int + featureVersion: Int, + + @Parameter(name = "release_types", description = "List of release types to include in computation (i.e &release_types=ga&release_types=ea)", required = false) + @QueryParam("release_types") + @DefaultValue("ga") + releaseTypes: List? ): Map { val release = apiDataStore.getAdoptRepos().getFeatureRelease(featureVersion) ?: throw BadRequestException("Unable to find version $featureVersion") - return getAdoptReleases(release) - .filter { it.release_type == ReleaseType.ga } + return getAdoptReleases(release, releaseTypes, null) .map { grouped -> Pair( grouped.release_name, @@ -89,20 +94,25 @@ class DownloadStatsResource { @GET @Schema(hidden = true) @Path("/total/{feature_version}/{release_name}") - @Operation(summary = "Get download stats for feature verson", description = "stats", hidden = true) + @Operation(summary = "Get download stats for feature version", description = "stats", hidden = true) fun getTotalDownloadStatsForTag( @Parameter(name = "feature_version", description = "Feature version (i.e 8, 9, 10...)", required = true) @PathParam("feature_version") featureVersion: Int, + @Parameter(name = "release_name", description = "Release Name i.e jdk-11.0.4+11", required = true) @PathParam("release_name") - releaseName: String + releaseName: String, + + @Parameter(name = "release_types", description = "List of release types to include in computation (i.e &release_types=ga&release_types=ea)", required = false) + @QueryParam("release_types") + @DefaultValue("ga") + releaseTypes: List? ): Map { val release = apiDataStore.getAdoptRepos().getFeatureRelease(featureVersion) ?: throw BadRequestException("Unable to find version $featureVersion") - return getAdoptReleases(release) - .filter { it.release_name == releaseName } + return getAdoptReleases(release, releaseTypes, releaseName) .flatMap { it.binaries.asSequence() } .flatMap { val archive = Pair(it.`package`.name, it.download_count) @@ -115,17 +125,27 @@ class DownloadStatsResource { .toMap() } - private fun getAdoptReleases(release: FeatureRelease): Sequence { - return release + private fun getAdoptReleases(release: FeatureRelease, releaseTypes: List?, releaseName: String?): Sequence { + var releases = release .releases .getReleases() .filter { it.vendor == Vendor.getDefault() } + + if(releaseTypes != null && releaseTypes.isNotEmpty()) { + releases = releases.filter { releaseTypes.contains(it.release_type) } + } + + if(releaseName != null) { + releases = releases.filter { it.release_name == releaseName } + } + + return releases } @GET @Schema(hidden = true) @Path("/tracking") - @Operation(summary = "Get download stats for feature verson", description = "stats", hidden = true) + @Operation(summary = "Get download stats for feature version", description = "stats", hidden = true) fun tracking( @Parameter(name = "days", description = "Number of days to display, if used in conjunction with from/to then this will limit the request to x days before the end of the given period", schema = Schema(defaultValue = "30", type = SchemaType.INTEGER), required = false) @QueryParam("days") @@ -165,7 +185,7 @@ class DownloadStatsResource { @GET @Schema(hidden = true) @Path("/monthly") - @Operation(summary = "Get download stats for feature verson", description = "stats", hidden = true) + @Operation(summary = "Get download stats for feature version", description = "stats", hidden = true) fun monthly( @Parameter(name = "source", description = "Stats data source", schema = Schema(defaultValue = "all"), required = false) @QueryParam("source") diff --git a/adoptium-frontend-parent/adoptium-api-v3-frontend/src/test/kotlin/net/adoptium/api/DownloadStatsPathTest.kt b/adoptium-frontend-parent/adoptium-api-v3-frontend/src/test/kotlin/net/adoptium/api/DownloadStatsPathTest.kt index 011de8147..947d05586 100644 --- a/adoptium-frontend-parent/adoptium-api-v3-frontend/src/test/kotlin/net/adoptium/api/DownloadStatsPathTest.kt +++ b/adoptium-frontend-parent/adoptium-api-v3-frontend/src/test/kotlin/net/adoptium/api/DownloadStatsPathTest.kt @@ -21,6 +21,7 @@ import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.ExtendWith import java.time.format.DateTimeFormatter import jakarta.ws.rs.BadRequestException +import net.adoptium.api.v3.models.ReleaseType @ExtendWith(value = [DbExtension::class]) @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -157,14 +158,14 @@ class DownloadStatsPathTest : FrontendTest() { @Test fun totalVersionReturnsSaneData() { - val stats = downloadStatsResource.getTotalDownloadStats(8) + val stats = downloadStatsResource.getTotalDownloadStats(8, listOf(ReleaseType.ga)) assertTrue { return@assertTrue stats.isNotEmpty() && !stats.containsValue(0L) } } @Test fun badTotalVersionReturnsSaneData() { assertThrows { - downloadStatsResource.getTotalDownloadStats(101) + downloadStatsResource.getTotalDownloadStats(101, listOf(ReleaseType.ga)) } } @@ -174,10 +175,11 @@ class DownloadStatsPathTest : FrontendTest() { val releases = getReleases() val release = releases + .filter { it.release_type == ReleaseType.ga } .filter { it.vendor == Vendor.getDefault() } .first() - val stats = downloadStatsResource.getTotalDownloadStatsForTag(release.version_data.major, release.release_name) + val stats = downloadStatsResource.getTotalDownloadStatsForTag(release.version_data.major, release.release_name, listOf(ReleaseType.ga)) assertTrue { return@assertTrue stats.isNotEmpty() && !stats.containsValue(0L) } } } @@ -185,7 +187,7 @@ class DownloadStatsPathTest : FrontendTest() { @Test fun badTotalTagReturnsSaneData() { assertThrows { - downloadStatsResource.getTotalDownloadStatsForTag(101, "fooBar") + downloadStatsResource.getTotalDownloadStatsForTag(101, "fooBar", listOf(ReleaseType.ga)) } } diff --git a/adoptium-frontend-parent/adoptium-frontend-assets/src/main/resources/META-INF/resources/favicon.ico b/adoptium-frontend-parent/adoptium-frontend-assets/src/main/resources/META-INF/resources/favicon.ico new file mode 100644 index 000000000..19260dad5 Binary files /dev/null and b/adoptium-frontend-parent/adoptium-frontend-assets/src/main/resources/META-INF/resources/favicon.ico differ diff --git a/adoptium-frontend-parent/adoptopenjdk-frontend-assets/src/main/resources/META-INF/resources/favicon.ico b/adoptium-frontend-parent/adoptopenjdk-frontend-assets/src/main/resources/META-INF/resources/favicon.ico new file mode 100644 index 000000000..6dcd214b5 Binary files /dev/null and b/adoptium-frontend-parent/adoptopenjdk-frontend-assets/src/main/resources/META-INF/resources/favicon.ico differ diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/application.properties b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/application.properties index ae811897b..cc774661d 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/application.properties +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/application.properties @@ -12,6 +12,8 @@ quarkus.log.category."org.mongodb".level=INFO quarkus.log.category."net.adoptium.api.v3.dataSources.APIDataStore".level=WARN quarkus.log.category."org.apache.http.client.protocol.ResponseProcessCookies".level=ERROR quarkus.log.category."io.netty".level=INFO +quarkus.log.category."org.kohsuke".level=WARN +quarkus.log.category."jdk.event.security".level=WARN quarkus.http.host=localhost diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/logback.xml b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/logback.xml index 64dfd08a5..a441e7b11 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/logback.xml +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/logback.xml @@ -5,6 +5,8 @@ + + /tmp/updater.log diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/GitHubAuthTest.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/GitHubAuthTest.kt index 8e75513e7..eb854d75b 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/GitHubAuthTest.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/GitHubAuthTest.kt @@ -50,14 +50,14 @@ class GitHubAuthTest { } @Test - fun `readToken prioritizes system property if env var is not defined`() { + suspend fun `readToken prioritizes system property if env var is not defined`() { assertFalse(System.getenv().containsKey(tokenKey)) val prevTokenProperty: String? = System.getProperty(tokenKey) System.setProperty(tokenKey, "system-property-token") try { - val actualToken = GitHubAuth.readToken() + val actualToken = GitHubAuth.getAuthenticationToken().token assertEquals("system-property-token", actualToken) } finally { if (prevTokenProperty == null) { @@ -69,7 +69,7 @@ class GitHubAuthTest { } @Test - fun `readToken falls back to property file if env var and system property are not defined`() { + suspend fun `readToken falls back to property file if env var and system property are not defined`() { assertFalse(System.getenv().containsKey(tokenKey)) assertFalse(System.getProperties().containsKey(tokenKey)) @@ -81,7 +81,7 @@ class GitHubAuthTest { } try { - val actualToken = GitHubAuth.readToken() + val actualToken = GitHubAuth.getAuthenticationToken().token assertEquals("real-file-token", actualToken) } finally { tokenDir.deleteRecursively() @@ -89,11 +89,11 @@ class GitHubAuthTest { } @Test - fun readsTokenNullFromFile() { + suspend fun readsTokenNullFromFile() { assertFalse(System.getenv().containsKey(tokenKey)) assertFalse(File(tempDir, ".adopt_api").exists()) - val actualToken = GitHubAuth.readToken() + val actualToken = GitHubAuth.getAuthenticationToken().token assertThat(actualToken, oneOf(null, "")) } } diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/pom.xml b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/pom.xml index e61ce282c..c468bd490 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/pom.xml +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/pom.xml @@ -16,6 +16,23 @@ net.adoptium.api adoptium-http-client-datasource + + io.jsonwebtoken + jjwt-api + 0.12.3 + + + io.jsonwebtoken + jjwt-impl + 0.12.3 + runtime + + + io.jsonwebtoken + jjwt-jackson + 0.12.3 + runtime + com.expediagroup graphql-kotlin-ktor-client @@ -30,6 +47,11 @@ com.expediagroup graphql-kotlin-client-jackson + + org.kohsuke + github-api + 1.317 + net.adoptium.api adoptium-api-v3-persistence diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubAuth.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubAuth.kt index 85f1ab5e0..354b9187b 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubAuth.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubAuth.kt @@ -1,22 +1,59 @@ package net.adoptium.api.v3.dataSources.github -import org.slf4j.LoggerFactory import java.io.File import java.nio.file.Files import java.util.Properties +import org.slf4j.LoggerFactory +import io.jsonwebtoken.Jwts +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.kohsuke.github.GHAppInstallation +import org.kohsuke.github.GHAppInstallationToken +import java.security.KeyFactory +import java.security.spec.PKCS8EncodedKeySpec +import java.util.Base64 +import java.util.Date +import org.kohsuke.github.GitHub +import org.kohsuke.github.GitHubBuilder class GitHubAuth { + data class AuthInfo(val token: String, val type: AuthType, val expirationTime: Date?) + enum class AuthType { + APP, TOKEN + } companion object { @JvmStatic private val LOGGER = LoggerFactory.getLogger(this::class.java) + private var TOKEN: AuthInfo? = null + private val appId = System.getenv("GITHUB_APP_ID") + private val privateKey = System.getenv("GITHUB_APP_PRIVATE_KEY") + private val installationId = System.getenv("GITHUB_APP_INSTALLATION_ID") + private val mutex = Mutex() - fun readToken(): String? { - var token = System.getenv("GITHUB_TOKEN") - if (token.isNullOrEmpty()) { - token = System.getProperty("GITHUB_TOKEN") - } + suspend fun getAuthenticationToken(): AuthInfo { + return mutex.withLock { + // Detect if we are using a GitHub App + if (!appId.isNullOrEmpty() && !privateKey.isNullOrEmpty() && !installationId.isNullOrEmpty()) { + if (TOKEN == null || (TOKEN!!.expirationTime != null && TOKEN!!.expirationTime!!.before(Date()))) { + LOGGER.info("Using GitHub App for authentication") + LOGGER.info("Generating a new installation token") + val token = authenticateAsGitHubApp(appId, privateKey, installationId) + TOKEN = AuthInfo(token.token, AuthType.APP, token.expiresAt) + } + } else { + if (TOKEN == null) { + val token = readToken() + LOGGER.info("Using Personal Access Token for authentication") + TOKEN = AuthInfo(token, AuthType.TOKEN, null) + } + } + TOKEN!! + } + } + private fun readToken(): String { + var token = System.getenv("GITHUB_TOKEN") if (token.isNullOrEmpty()) { val userHome = System.getProperty("user.home") @@ -33,8 +70,45 @@ class GitHubAuth { } if (token.isNullOrEmpty()) { LOGGER.error("Could not find GITHUB_TOKEN") + throw FailedToAuthenticateException() } return token } + + private suspend fun authenticateAsGitHubApp(appId: String, privateKey: String, installationId: String): GHAppInstallationToken { + try { + // Remove the first and last lines + val sanitizedKey = privateKey + .replace("-----BEGIN PRIVATE KEY-----", "") + .replace("-----END PRIVATE KEY-----", "") + .replace("\\s".toRegex(), "") + + // Decode the Base64 encoded key + val keyBytes = Base64.getDecoder().decode(sanitizedKey) + + // Generate the private key + val keySpec = PKCS8EncodedKeySpec(keyBytes) + val keyFactory = KeyFactory.getInstance("RSA") + val privateKey = keyFactory.generatePrivate(keySpec) + + // Create and sign the JWT + val nowMillis = System.currentTimeMillis() + val jwtToken = Jwts.builder() + .issuer(appId) + .issuedAt(Date(nowMillis)) + .expiration(Date(nowMillis + 60000)) // Token valid for 1 minute + .signWith(privateKey, Jwts.SIG.RS256) + .compact() + + val gitHubApp: GitHub = GitHubBuilder().withJwtToken(jwtToken).build() + val appInstallation: GHAppInstallation = gitHubApp.getApp().getInstallationById(installationId.toLong()) + return appInstallation.createToken().create() + } catch (e: Exception) { + LOGGER.error("Error authenticating as GitHub App", e) + throw FailedToAuthenticateException() + } + } } + + class FailedToAuthenticateException : Exception("Failed to authenticate to GitHub") {} } diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLRequestImpl.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLRequestImpl.kt index 8b34e7578..ac2ea79f3 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLRequestImpl.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLRequestImpl.kt @@ -8,6 +8,7 @@ import io.ktor.client.* import jakarta.enterprise.context.ApplicationScoped import net.adoptium.api.v3.dataSources.UpdaterJsonMapper import net.adoptium.api.v3.dataSources.github.GitHubAuth +import net.adoptium.api.v3.dataSources.github.GitHubAuth.AuthInfo import java.net.URL @ApplicationScoped @@ -16,15 +17,8 @@ open class GraphQLRequestImpl : GraphQLRequest { private val client: GraphQLKtorClient private val httpClient: HttpClient val BASE_URL = "https://api.github.com/graphql" - private val TOKEN: String init { - val token = GitHubAuth.readToken() - if (token == null) { - throw IllegalStateException("No token provided") - } else { - TOKEN = token - } httpClient = HttpClient() client = GraphQLKtorClient( url = URL(BASE_URL), @@ -34,8 +28,9 @@ open class GraphQLRequestImpl : GraphQLRequest { } override suspend fun request(query: GraphQLClientRequest): GraphQLClientResponse { + val authInfo: AuthInfo = GitHubAuth.getAuthenticationToken() return client.execute(query) { - headers.append("Authorization", "Bearer $TOKEN") + headers.append("Authorization", "Bearer ${authInfo.token}") } } } diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml index f6daf7478..7d6657284 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml @@ -48,6 +48,28 @@ org.slf4j slf4j-api + + io.jsonwebtoken + jjwt-api + 0.12.3 + + + io.jsonwebtoken + jjwt-impl + 0.12.3 + runtime + + + io.jsonwebtoken + jjwt-jackson + 0.12.3 + runtime + + + org.kohsuke + github-api + 1.317 + diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/DefaultUpdaterHtmlClient.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/DefaultUpdaterHtmlClient.kt index 1187d4a1e..07e75b73e 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/DefaultUpdaterHtmlClient.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/DefaultUpdaterHtmlClient.kt @@ -19,6 +19,7 @@ import kotlin.coroutines.Continuation import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException import kotlin.coroutines.suspendCoroutine +import net.adoptium.api.v3.dataSources.GitHubAuth.AuthInfo @Default @ApplicationScoped @@ -33,7 +34,6 @@ open class DefaultUpdaterHtmlClient @Inject constructor( companion object { @JvmStatic private val LOGGER = LoggerFactory.getLogger(this::class.java) - private val TOKEN: String? = GitHubAuth.readToken() private const val REQUEST_TIMEOUT = 12_000L private val GITHUB_DOMAINS = listOf("api.github.com", "github.com") @@ -50,7 +50,8 @@ open class DefaultUpdaterHtmlClient @Inject constructor( class ResponseHandler( val client: DefaultUpdaterHtmlClient, private val continuation: Continuation, - val request: UrlRequest? + val request: UrlRequest?, + val token: String? ) : FutureCallback { override fun cancelled() { continuation.resumeWithException(Exception("cancelled")) @@ -64,7 +65,7 @@ open class DefaultUpdaterHtmlClient @Inject constructor( } isARedirect(response) -> { - client.getData(UrlRequest(response.getFirstHeader("location").value, request?.lastModified), continuation) + client.getData(UrlRequest(response.getFirstHeader("location").value, request?.lastModified), continuation, token) } response.statusLine.statusCode == 404 -> { @@ -93,14 +94,14 @@ open class DefaultUpdaterHtmlClient @Inject constructor( override fun failed(e: java.lang.Exception?) { if (e == null) { - continuation.resumeWithException(Exception("Failed Uknown reason")) + continuation.resumeWithException(Exception("Failed Unknown reason")) } else { continuation.resumeWithException(e) } } } - private fun getData(urlRequest: UrlRequest, continuation: Continuation) { + private fun getData(urlRequest: UrlRequest, continuation: Continuation, token: String?) { try { val url = URL(urlRequest.url) val request = RequestBuilder @@ -112,8 +113,8 @@ open class DefaultUpdaterHtmlClient @Inject constructor( request.addHeader("If-Modified-Since", urlRequest.lastModified) } - if (GITHUB_DOMAINS.contains(url.host) && TOKEN != null) { - request.setHeader("Authorization", "token $TOKEN") + if (token != null && GITHUB_DOMAINS.contains(url.host)) { + request.setHeader("Authorization", "token $token") } val client = @@ -123,20 +124,25 @@ open class DefaultUpdaterHtmlClient @Inject constructor( redirectingHttpClient } - client.execute(request, ResponseHandler(this, continuation, urlRequest)) + client.execute(request, ResponseHandler(this, continuation, urlRequest, token)) } catch (e: Exception) { continuation.resumeWith(Result.failure(e)) } } override suspend fun getFullResponse(request: UrlRequest): HttpResponse? { + val requestURL = URL(request.url) + var authInfo: AuthInfo? = null + if (GITHUB_DOMAINS.contains(requestURL.host)) { + authInfo = GitHubAuth.getAuthenticationToken() + } // Retry up to 10 times for (retryCount in 1..10) { try { LOGGER.debug("Getting ${request.url} ${request.lastModified}") val response: HttpResponse = withTimeout(REQUEST_TIMEOUT) { suspendCoroutine { continuation -> - getData(request, continuation) + getData(request, continuation, authInfo?.token) } } LOGGER.debug("Got ${request.url}") diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/GitHubAuth.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/GitHubAuth.kt index 76892930f..c2cc24a74 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/GitHubAuth.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/GitHubAuth.kt @@ -1,22 +1,57 @@ package net.adoptium.api.v3.dataSources +import io.jsonwebtoken.Jwts +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.kohsuke.github.GHAppInstallation +import org.kohsuke.github.GHAppInstallationToken +import org.kohsuke.github.GitHub +import org.kohsuke.github.GitHubBuilder import org.slf4j.LoggerFactory import java.io.File import java.nio.file.Files -import java.util.Properties +import java.security.KeyFactory +import java.security.spec.PKCS8EncodedKeySpec +import java.util.* class GitHubAuth { + data class AuthInfo(val token: String, val type: AuthType, val expirationTime: Date?) + enum class AuthType { + APP, TOKEN + } companion object { @JvmStatic private val LOGGER = LoggerFactory.getLogger(this::class.java) + private var TOKEN: AuthInfo? = null + private val appId = System.getenv("GITHUB_APP_ID") + private val privateKey = System.getenv("GITHUB_APP_PRIVATE_KEY") + private val installationId = System.getenv("GITHUB_APP_INSTALLATION_ID") + private val mutex = Mutex() - fun readToken(): String? { - var token = System.getenv("GITHUB_TOKEN") - if (token.isNullOrEmpty()) { - token = System.getProperty("GITHUB_TOKEN") + suspend fun getAuthenticationToken(): AuthInfo { + return mutex.withLock { + // Detect if we are using a GitHub App + if (!appId.isNullOrEmpty() && !privateKey.isNullOrEmpty() && !installationId.isNullOrEmpty()) { + if (TOKEN == null || (TOKEN!!.expirationTime != null && TOKEN!!.expirationTime!!.before(Date()))) { + LOGGER.info("Using GitHub App for authentication") + LOGGER.info("Generating a new installation token") + val token = authenticateAsGitHubApp(appId, privateKey, installationId) + TOKEN = AuthInfo(token.token, AuthType.APP, token.expiresAt) + } + } else { + if (TOKEN == null) { + val token = readToken() + LOGGER.info("Using Personal Access Token for authentication") + TOKEN = AuthInfo(token, AuthType.TOKEN, null) + } + } + TOKEN!! } + } + private fun readToken(): String { + var token = System.getenv("GITHUB_TOKEN") if (token.isNullOrEmpty()) { val userHome = System.getProperty("user.home") @@ -33,8 +68,45 @@ class GitHubAuth { } if (token.isNullOrEmpty()) { LOGGER.error("Could not find GITHUB_TOKEN") + throw FailedToAuthenticateException() } return token } + + private suspend fun authenticateAsGitHubApp(appId: String, privateKey: String, installationId: String): GHAppInstallationToken { + try { + // Remove the first and last lines + val sanitizedKey = privateKey + .replace("-----BEGIN PRIVATE KEY-----", "") + .replace("-----END PRIVATE KEY-----", "") + .replace("\\s".toRegex(), "") + + // Decode the Base64 encoded key + val keyBytes = Base64.getDecoder().decode(sanitizedKey) + + // Generate the private key + val keySpec = PKCS8EncodedKeySpec(keyBytes) + val keyFactory = KeyFactory.getInstance("RSA") + val privateKey = keyFactory.generatePrivate(keySpec) + + // Create and sign the JWT + val nowMillis = System.currentTimeMillis() + val jwtToken = Jwts.builder() + .issuer(appId) + .issuedAt(Date(nowMillis)) + .expiration(Date(nowMillis + 60000)) // Token valid for 1 minute + .signWith(privateKey, Jwts.SIG.RS256) + .compact() + + val gitHubApp: GitHub = GitHubBuilder().withJwtToken(jwtToken).build() + val appInstallation: GHAppInstallation = gitHubApp.getApp().getInstallationById(installationId.toLong()) + return appInstallation.createToken().create() + } catch (e: Exception) { + LOGGER.error("Error authenticating as GitHub App", e) + throw FailedToAuthenticateException() + } + } } + + class FailedToAuthenticateException : Exception("Failed to authenticate as GitHub App") {} } diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 5b7084877..680c99a3d 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,4 +1,4 @@ -FROM eclipse-temurin:17 +FROM eclipse-temurin:21 # BUILDS FOR ADOPTIUM, NOT ADOPTOPENJDK ARG MAVEN_FLAGS="-Padoptium,-adoptopenjdk" diff --git a/docker-compose.yml b/docker-compose.yml index 1d7bce14b..a98a67e0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,4 +26,6 @@ services: environment: MONGODB_HOST: mongodb GITHUB_TOKEN: "${GITHUB_TOKEN}" - + GITHUB_APP_ID: "${GITHUB_APP_ID}" + GITHUB_APP_PRIVATE_KEY: "${GITHUB_APP_PRIVATE_KEY}" + GITHUB_APP_INSTALLATION_ID: "${GITHUB_APP_INSTALLATION_ID}" diff --git a/pom.xml b/pom.xml index acc743761..8d0928d69 100644 --- a/pom.xml +++ b/pom.xml @@ -10,29 +10,29 @@ - 3.9.2 + 3.9.5 1.7.1 - 0.8.10 + 0.8.11 17 17 - 1.9.10 + 1.9.20 3.1.0 3.6.0 3.4.0 2.12.1 3.3.0 - 3.3.1 + 3.3.2 3.11.0 17 17 17 17 17 - 3.6.0 + 3.6.1 2.0.4 3.1.1 1.0 @@ -50,29 +50,29 @@ 0.8.10 3.3.0 - 3.6.0 + 3.6.2 2.0 3.0.0-alpha-1 3.1.2 - 3.3.0 - 2.2.0 + 3.3.1 + 2.3.0 7.0.1 - 1.15.0 - 3.21.0 + 1.15.3 + 3.21.2 3.4.5 0.15 3.0.1 3.3.1 2.0.1 - 4.0.0-M10 + 4.0.0-M11 3.3.0 - 4.7.3.6 - 3.1.2 + 4.8.1.0 + 3.2.2 1.0.10 1.2.0 3.1.0 - 3.9.2 + 3.9.5 2.16.1 3.4.0 0.9.1 @@ -82,7 +82,7 @@ 4.7.3 UTF-8 UTF-8 - 3.4.2 + 3.5.1 diff --git a/speccy.yaml b/speccy.yaml deleted file mode 100644 index 2435628b5..000000000 --- a/speccy.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Output a lot of information about what is happening (wont work if you have quiet on) -# Default stdout = 1 -verbose: 2 -# Rules specific to the lint command -lint: - # rules files to load - rules: - - strict - # rules to skip - skip: - - info-contact - - operation-operationId - - short-summary - - tag-description \ No newline at end of file