diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a5a966516..4f66ec001 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,6 +34,24 @@ If you want to use the updater tool to add entries into the database, you need t The production server uses mongodb to store data, however you can also use Fongo. If you would like to install mongodb and are on mac, I used this [guide](https://zellwk.com/blog/install-mongodb/) which utilizes homebrew. You can also install `mongo` which is a command-line tool that gives you access to your mongodb, allowing you to manually search through the database. +### GitHub App Authentication + +The updater can be used with a GitHub Token or GitHub App. To use a GitHub app you need to generate an app on GitHub. Once you've done that you need to convert the key to PKCS#8 format using the following command: + +```bash +openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in your-rsa-private-key.pem -out pkcs8-key.pem +``` + +Once this is done you can export the following variables at runtime: + +```bash +export GITHUB_APP_ID="1234" +export GITHUB_APP_INSTALLATION_ID="1234" +export GITHUB_APP_PRIVATE_KEY=$'-----BEGIN PRIVATE KEY----- + +-----END PRIVATE KEY-----' +``` + ### Build Tool [Maven](https://maven.apache.org/index.html) is used to build the project. diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/application.properties b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/application.properties index ae811897b..cc774661d 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/application.properties +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/application.properties @@ -12,6 +12,8 @@ quarkus.log.category."org.mongodb".level=INFO quarkus.log.category."net.adoptium.api.v3.dataSources.APIDataStore".level=WARN quarkus.log.category."org.apache.http.client.protocol.ResponseProcessCookies".level=ERROR quarkus.log.category."io.netty".level=INFO +quarkus.log.category."org.kohsuke".level=WARN +quarkus.log.category."jdk.event.security".level=WARN quarkus.http.host=localhost diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/logback.xml b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/logback.xml index 64dfd08a5..a441e7b11 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/logback.xml +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/resources/logback.xml @@ -5,6 +5,8 @@ + + /tmp/updater.log diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/GitHubAuthTest.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/GitHubAuthTest.kt index 8e75513e7..eb854d75b 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/GitHubAuthTest.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/GitHubAuthTest.kt @@ -50,14 +50,14 @@ class GitHubAuthTest { } @Test - fun `readToken prioritizes system property if env var is not defined`() { + suspend fun `readToken prioritizes system property if env var is not defined`() { assertFalse(System.getenv().containsKey(tokenKey)) val prevTokenProperty: String? = System.getProperty(tokenKey) System.setProperty(tokenKey, "system-property-token") try { - val actualToken = GitHubAuth.readToken() + val actualToken = GitHubAuth.getAuthenticationToken().token assertEquals("system-property-token", actualToken) } finally { if (prevTokenProperty == null) { @@ -69,7 +69,7 @@ class GitHubAuthTest { } @Test - fun `readToken falls back to property file if env var and system property are not defined`() { + suspend fun `readToken falls back to property file if env var and system property are not defined`() { assertFalse(System.getenv().containsKey(tokenKey)) assertFalse(System.getProperties().containsKey(tokenKey)) @@ -81,7 +81,7 @@ class GitHubAuthTest { } try { - val actualToken = GitHubAuth.readToken() + val actualToken = GitHubAuth.getAuthenticationToken().token assertEquals("real-file-token", actualToken) } finally { tokenDir.deleteRecursively() @@ -89,11 +89,11 @@ class GitHubAuthTest { } @Test - fun readsTokenNullFromFile() { + suspend fun readsTokenNullFromFile() { assertFalse(System.getenv().containsKey(tokenKey)) assertFalse(File(tempDir, ".adopt_api").exists()) - val actualToken = GitHubAuth.readToken() + val actualToken = GitHubAuth.getAuthenticationToken().token assertThat(actualToken, oneOf(null, "")) } } diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/pom.xml b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/pom.xml index e61ce282c..c468bd490 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/pom.xml +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/pom.xml @@ -16,6 +16,23 @@ net.adoptium.api adoptium-http-client-datasource + + io.jsonwebtoken + jjwt-api + 0.12.3 + + + io.jsonwebtoken + jjwt-impl + 0.12.3 + runtime + + + io.jsonwebtoken + jjwt-jackson + 0.12.3 + runtime + com.expediagroup graphql-kotlin-ktor-client @@ -30,6 +47,11 @@ com.expediagroup graphql-kotlin-client-jackson + + org.kohsuke + github-api + 1.317 + net.adoptium.api adoptium-api-v3-persistence diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubAuth.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubAuth.kt index 85f1ab5e0..354b9187b 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubAuth.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubAuth.kt @@ -1,22 +1,59 @@ package net.adoptium.api.v3.dataSources.github -import org.slf4j.LoggerFactory import java.io.File import java.nio.file.Files import java.util.Properties +import org.slf4j.LoggerFactory +import io.jsonwebtoken.Jwts +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.kohsuke.github.GHAppInstallation +import org.kohsuke.github.GHAppInstallationToken +import java.security.KeyFactory +import java.security.spec.PKCS8EncodedKeySpec +import java.util.Base64 +import java.util.Date +import org.kohsuke.github.GitHub +import org.kohsuke.github.GitHubBuilder class GitHubAuth { + data class AuthInfo(val token: String, val type: AuthType, val expirationTime: Date?) + enum class AuthType { + APP, TOKEN + } companion object { @JvmStatic private val LOGGER = LoggerFactory.getLogger(this::class.java) + private var TOKEN: AuthInfo? = null + private val appId = System.getenv("GITHUB_APP_ID") + private val privateKey = System.getenv("GITHUB_APP_PRIVATE_KEY") + private val installationId = System.getenv("GITHUB_APP_INSTALLATION_ID") + private val mutex = Mutex() - fun readToken(): String? { - var token = System.getenv("GITHUB_TOKEN") - if (token.isNullOrEmpty()) { - token = System.getProperty("GITHUB_TOKEN") - } + suspend fun getAuthenticationToken(): AuthInfo { + return mutex.withLock { + // Detect if we are using a GitHub App + if (!appId.isNullOrEmpty() && !privateKey.isNullOrEmpty() && !installationId.isNullOrEmpty()) { + if (TOKEN == null || (TOKEN!!.expirationTime != null && TOKEN!!.expirationTime!!.before(Date()))) { + LOGGER.info("Using GitHub App for authentication") + LOGGER.info("Generating a new installation token") + val token = authenticateAsGitHubApp(appId, privateKey, installationId) + TOKEN = AuthInfo(token.token, AuthType.APP, token.expiresAt) + } + } else { + if (TOKEN == null) { + val token = readToken() + LOGGER.info("Using Personal Access Token for authentication") + TOKEN = AuthInfo(token, AuthType.TOKEN, null) + } + } + TOKEN!! + } + } + private fun readToken(): String { + var token = System.getenv("GITHUB_TOKEN") if (token.isNullOrEmpty()) { val userHome = System.getProperty("user.home") @@ -33,8 +70,45 @@ class GitHubAuth { } if (token.isNullOrEmpty()) { LOGGER.error("Could not find GITHUB_TOKEN") + throw FailedToAuthenticateException() } return token } + + private suspend fun authenticateAsGitHubApp(appId: String, privateKey: String, installationId: String): GHAppInstallationToken { + try { + // Remove the first and last lines + val sanitizedKey = privateKey + .replace("-----BEGIN PRIVATE KEY-----", "") + .replace("-----END PRIVATE KEY-----", "") + .replace("\\s".toRegex(), "") + + // Decode the Base64 encoded key + val keyBytes = Base64.getDecoder().decode(sanitizedKey) + + // Generate the private key + val keySpec = PKCS8EncodedKeySpec(keyBytes) + val keyFactory = KeyFactory.getInstance("RSA") + val privateKey = keyFactory.generatePrivate(keySpec) + + // Create and sign the JWT + val nowMillis = System.currentTimeMillis() + val jwtToken = Jwts.builder() + .issuer(appId) + .issuedAt(Date(nowMillis)) + .expiration(Date(nowMillis + 60000)) // Token valid for 1 minute + .signWith(privateKey, Jwts.SIG.RS256) + .compact() + + val gitHubApp: GitHub = GitHubBuilder().withJwtToken(jwtToken).build() + val appInstallation: GHAppInstallation = gitHubApp.getApp().getInstallationById(installationId.toLong()) + return appInstallation.createToken().create() + } catch (e: Exception) { + LOGGER.error("Error authenticating as GitHub App", e) + throw FailedToAuthenticateException() + } + } } + + class FailedToAuthenticateException : Exception("Failed to authenticate to GitHub") {} } diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLRequestImpl.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLRequestImpl.kt index 8b34e7578..ac2ea79f3 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLRequestImpl.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLRequestImpl.kt @@ -8,6 +8,7 @@ import io.ktor.client.* import jakarta.enterprise.context.ApplicationScoped import net.adoptium.api.v3.dataSources.UpdaterJsonMapper import net.adoptium.api.v3.dataSources.github.GitHubAuth +import net.adoptium.api.v3.dataSources.github.GitHubAuth.AuthInfo import java.net.URL @ApplicationScoped @@ -16,15 +17,8 @@ open class GraphQLRequestImpl : GraphQLRequest { private val client: GraphQLKtorClient private val httpClient: HttpClient val BASE_URL = "https://api.github.com/graphql" - private val TOKEN: String init { - val token = GitHubAuth.readToken() - if (token == null) { - throw IllegalStateException("No token provided") - } else { - TOKEN = token - } httpClient = HttpClient() client = GraphQLKtorClient( url = URL(BASE_URL), @@ -34,8 +28,9 @@ open class GraphQLRequestImpl : GraphQLRequest { } override suspend fun request(query: GraphQLClientRequest): GraphQLClientResponse { + val authInfo: AuthInfo = GitHubAuth.getAuthenticationToken() return client.execute(query) { - headers.append("Authorization", "Bearer $TOKEN") + headers.append("Authorization", "Bearer ${authInfo.token}") } } } diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml index f6daf7478..7d6657284 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml @@ -48,6 +48,28 @@ org.slf4j slf4j-api + + io.jsonwebtoken + jjwt-api + 0.12.3 + + + io.jsonwebtoken + jjwt-impl + 0.12.3 + runtime + + + io.jsonwebtoken + jjwt-jackson + 0.12.3 + runtime + + + org.kohsuke + github-api + 1.317 + diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/DefaultUpdaterHtmlClient.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/DefaultUpdaterHtmlClient.kt index 1187d4a1e..07e75b73e 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/DefaultUpdaterHtmlClient.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/DefaultUpdaterHtmlClient.kt @@ -19,6 +19,7 @@ import kotlin.coroutines.Continuation import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException import kotlin.coroutines.suspendCoroutine +import net.adoptium.api.v3.dataSources.GitHubAuth.AuthInfo @Default @ApplicationScoped @@ -33,7 +34,6 @@ open class DefaultUpdaterHtmlClient @Inject constructor( companion object { @JvmStatic private val LOGGER = LoggerFactory.getLogger(this::class.java) - private val TOKEN: String? = GitHubAuth.readToken() private const val REQUEST_TIMEOUT = 12_000L private val GITHUB_DOMAINS = listOf("api.github.com", "github.com") @@ -50,7 +50,8 @@ open class DefaultUpdaterHtmlClient @Inject constructor( class ResponseHandler( val client: DefaultUpdaterHtmlClient, private val continuation: Continuation, - val request: UrlRequest? + val request: UrlRequest?, + val token: String? ) : FutureCallback { override fun cancelled() { continuation.resumeWithException(Exception("cancelled")) @@ -64,7 +65,7 @@ open class DefaultUpdaterHtmlClient @Inject constructor( } isARedirect(response) -> { - client.getData(UrlRequest(response.getFirstHeader("location").value, request?.lastModified), continuation) + client.getData(UrlRequest(response.getFirstHeader("location").value, request?.lastModified), continuation, token) } response.statusLine.statusCode == 404 -> { @@ -93,14 +94,14 @@ open class DefaultUpdaterHtmlClient @Inject constructor( override fun failed(e: java.lang.Exception?) { if (e == null) { - continuation.resumeWithException(Exception("Failed Uknown reason")) + continuation.resumeWithException(Exception("Failed Unknown reason")) } else { continuation.resumeWithException(e) } } } - private fun getData(urlRequest: UrlRequest, continuation: Continuation) { + private fun getData(urlRequest: UrlRequest, continuation: Continuation, token: String?) { try { val url = URL(urlRequest.url) val request = RequestBuilder @@ -112,8 +113,8 @@ open class DefaultUpdaterHtmlClient @Inject constructor( request.addHeader("If-Modified-Since", urlRequest.lastModified) } - if (GITHUB_DOMAINS.contains(url.host) && TOKEN != null) { - request.setHeader("Authorization", "token $TOKEN") + if (token != null && GITHUB_DOMAINS.contains(url.host)) { + request.setHeader("Authorization", "token $token") } val client = @@ -123,20 +124,25 @@ open class DefaultUpdaterHtmlClient @Inject constructor( redirectingHttpClient } - client.execute(request, ResponseHandler(this, continuation, urlRequest)) + client.execute(request, ResponseHandler(this, continuation, urlRequest, token)) } catch (e: Exception) { continuation.resumeWith(Result.failure(e)) } } override suspend fun getFullResponse(request: UrlRequest): HttpResponse? { + val requestURL = URL(request.url) + var authInfo: AuthInfo? = null + if (GITHUB_DOMAINS.contains(requestURL.host)) { + authInfo = GitHubAuth.getAuthenticationToken() + } // Retry up to 10 times for (retryCount in 1..10) { try { LOGGER.debug("Getting ${request.url} ${request.lastModified}") val response: HttpResponse = withTimeout(REQUEST_TIMEOUT) { suspendCoroutine { continuation -> - getData(request, continuation) + getData(request, continuation, authInfo?.token) } } LOGGER.debug("Got ${request.url}") diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/GitHubAuth.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/GitHubAuth.kt index 76892930f..c2cc24a74 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/GitHubAuth.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/GitHubAuth.kt @@ -1,22 +1,57 @@ package net.adoptium.api.v3.dataSources +import io.jsonwebtoken.Jwts +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.kohsuke.github.GHAppInstallation +import org.kohsuke.github.GHAppInstallationToken +import org.kohsuke.github.GitHub +import org.kohsuke.github.GitHubBuilder import org.slf4j.LoggerFactory import java.io.File import java.nio.file.Files -import java.util.Properties +import java.security.KeyFactory +import java.security.spec.PKCS8EncodedKeySpec +import java.util.* class GitHubAuth { + data class AuthInfo(val token: String, val type: AuthType, val expirationTime: Date?) + enum class AuthType { + APP, TOKEN + } companion object { @JvmStatic private val LOGGER = LoggerFactory.getLogger(this::class.java) + private var TOKEN: AuthInfo? = null + private val appId = System.getenv("GITHUB_APP_ID") + private val privateKey = System.getenv("GITHUB_APP_PRIVATE_KEY") + private val installationId = System.getenv("GITHUB_APP_INSTALLATION_ID") + private val mutex = Mutex() - fun readToken(): String? { - var token = System.getenv("GITHUB_TOKEN") - if (token.isNullOrEmpty()) { - token = System.getProperty("GITHUB_TOKEN") + suspend fun getAuthenticationToken(): AuthInfo { + return mutex.withLock { + // Detect if we are using a GitHub App + if (!appId.isNullOrEmpty() && !privateKey.isNullOrEmpty() && !installationId.isNullOrEmpty()) { + if (TOKEN == null || (TOKEN!!.expirationTime != null && TOKEN!!.expirationTime!!.before(Date()))) { + LOGGER.info("Using GitHub App for authentication") + LOGGER.info("Generating a new installation token") + val token = authenticateAsGitHubApp(appId, privateKey, installationId) + TOKEN = AuthInfo(token.token, AuthType.APP, token.expiresAt) + } + } else { + if (TOKEN == null) { + val token = readToken() + LOGGER.info("Using Personal Access Token for authentication") + TOKEN = AuthInfo(token, AuthType.TOKEN, null) + } + } + TOKEN!! } + } + private fun readToken(): String { + var token = System.getenv("GITHUB_TOKEN") if (token.isNullOrEmpty()) { val userHome = System.getProperty("user.home") @@ -33,8 +68,45 @@ class GitHubAuth { } if (token.isNullOrEmpty()) { LOGGER.error("Could not find GITHUB_TOKEN") + throw FailedToAuthenticateException() } return token } + + private suspend fun authenticateAsGitHubApp(appId: String, privateKey: String, installationId: String): GHAppInstallationToken { + try { + // Remove the first and last lines + val sanitizedKey = privateKey + .replace("-----BEGIN PRIVATE KEY-----", "") + .replace("-----END PRIVATE KEY-----", "") + .replace("\\s".toRegex(), "") + + // Decode the Base64 encoded key + val keyBytes = Base64.getDecoder().decode(sanitizedKey) + + // Generate the private key + val keySpec = PKCS8EncodedKeySpec(keyBytes) + val keyFactory = KeyFactory.getInstance("RSA") + val privateKey = keyFactory.generatePrivate(keySpec) + + // Create and sign the JWT + val nowMillis = System.currentTimeMillis() + val jwtToken = Jwts.builder() + .issuer(appId) + .issuedAt(Date(nowMillis)) + .expiration(Date(nowMillis + 60000)) // Token valid for 1 minute + .signWith(privateKey, Jwts.SIG.RS256) + .compact() + + val gitHubApp: GitHub = GitHubBuilder().withJwtToken(jwtToken).build() + val appInstallation: GHAppInstallation = gitHubApp.getApp().getInstallationById(installationId.toLong()) + return appInstallation.createToken().create() + } catch (e: Exception) { + LOGGER.error("Error authenticating as GitHub App", e) + throw FailedToAuthenticateException() + } + } } + + class FailedToAuthenticateException : Exception("Failed to authenticate as GitHub App") {} } diff --git a/docker-compose.yml b/docker-compose.yml index 1d7bce14b..a98a67e0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,4 +26,6 @@ services: environment: MONGODB_HOST: mongodb GITHUB_TOKEN: "${GITHUB_TOKEN}" - + GITHUB_APP_ID: "${GITHUB_APP_ID}" + GITHUB_APP_PRIVATE_KEY: "${GITHUB_APP_PRIVATE_KEY}" + GITHUB_APP_INSTALLATION_ID: "${GITHUB_APP_INSTALLATION_ID}"