From 415f64198a6db10688b567cb38af5bf814fbf2e5 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Tue, 25 Jul 2023 03:38:55 +0900 Subject: [PATCH 01/17] Adds cross platform architecture compilation for native image --- .github/workflows/validate-pr.yml | 2 +- build.sbt | 2 +- project/build.properties | 2 +- .../GraalVMNativeImageKeys.scala | 4 + .../GraalVMNativeImagePlugin.scala | 88 +++++++++++-------- .../docker-native-image-arm64/build.sbt | 7 ++ .../project/plugins.sbt | 1 + .../src/main/scala/Main.scala | 5 ++ .../docker-native-image-arm64/test | 3 + .../docker-native-image/build.sbt | 3 +- .../simple-native-image/build.sbt | 2 +- 11 files changed, 79 insertions(+), 40 deletions(-) create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-arm64/project/plugins.sbt create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-arm64/src/main/scala/Main.scala create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-arm64/test diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml index e2eddd119..c70d8b646 100644 --- a/.github/workflows/validate-pr.yml +++ b/.github/workflows/validate-pr.yml @@ -254,7 +254,7 @@ jobs: - name: Setup GraalVM environment uses: olafurpg/setup-scala@v10 with: - java-version: graalvm@22.0.0=tgz+https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-22.0.0.2/graalvm-ce-java11-linux-amd64-22.0.0.2.tar.gz + java-version: graalvm@22.3.2=tgz+https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-22.3.2/graalvm-ce-java17-linux-amd64-22.3.2.tar.gz - name: Install native-image run: gu install native-image - name: Validate diff --git a/build.sbt b/build.sbt index 489c4149b..4cab8248e 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ organization := "com.github.sbt" homepage := Some(url("https://github.com/sbt/sbt-native-packager")) Global / onChangedBuildSource := ReloadOnSourceChanges -Global / scalaVersion := "2.12.12" +Global / scalaVersion := "2.12.13" // crossBuildingSettings crossSbtVersions := Vector("1.1.6") diff --git a/project/build.properties b/project/build.properties index 9edb75b77..52413ab79 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.4 +sbt.version=1.9.3 diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImageKeys.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImageKeys.scala index 8460b265d..b3e2aa8ea 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImageKeys.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImageKeys.scala @@ -14,6 +14,10 @@ trait GraalVMNativeImageKeys { val graalVMNativeImageGraalVersion = settingKey[Option[String]]( "Version of GraalVM to build with. Setting this has the effect of generating a container build image to build the native image with this version of GraalVM." ) + + val graalVMNativeImagePlatformArch = settingKey[Option[String]]( + "Platform architecture of GraalVM to build with. This only works when building the native image with a container." + ) } trait GraalVMNativeImageKeysEx extends GraalVMNativeImageKeys { diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala index c3ba42ab3..e65b71c25 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala @@ -37,6 +37,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { target in GraalVMNativeImage := target.value / "graalvm-native-image", graalVMNativeImageOptions := Seq.empty, graalVMNativeImageGraalVersion := None, + graalVMNativeImagePlatformArch := None, graalVMNativeImageCommand := (if (scala.util.Properties.isWin) "native-image.cmd" else "native-image"), resourceDirectory in GraalVMNativeImage := sourceDirectory.value / "graal", mainClass in GraalVMNativeImage := (mainClass in Compile).value @@ -48,7 +49,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { resources := resourceDirectories.value.descendantsExcept(includeFilter.value, excludeFilter.value).get, UniversalPlugin.autoImport.containerBuildImage := Def.taskDyn { graalVMNativeImageGraalVersion.value match { - case Some(tag) => generateContainerBuildImage(s"$GraalVMBaseImage:$tag") + case Some(tag) => generateContainerBuildImage(s"$GraalVMBaseImage:$tag", graalVMNativeImagePlatformArch.value) case None => Def.task(None: Option[String]) } }.value, @@ -59,6 +60,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { val className = mainClass.value.getOrElse(sys.error("Could not find a main class.")) val classpathJars = scriptClasspathOrdering.value val extraOptions = graalVMNativeImageOptions.value + val platformArch = graalVMNativeImagePlatformArch.value val streams = Keys.streams.value val dockerCommand = DockerPlugin.autoImport.dockerExecCommand.value val graalResourceDirectories = resourceDirectories.value @@ -85,6 +87,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { className, classpathJars, extraOptions, + platformArch, dockerCommand, resourceMappings, image, @@ -94,15 +97,13 @@ object GraalVMNativeImagePlugin extends AutoPlugin { } ) - private def buildLocal( - targetDirectory: File, - binaryName: String, - nativeImageCommand: String, - className: String, - classpathJars: Seq[File], - extraOptions: Seq[String], - log: ProcessLogger - ): File = { + private def buildLocal(targetDirectory: File, + binaryName: String, + nativeImageCommand: String, + className: String, + classpathJars: Seq[File], + extraOptions: Seq[String], + log: ProcessLogger): File = { targetDirectory.mkdirs() val command = { val nativeImageArguments = { @@ -112,8 +113,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { IO.withTemporaryFile("native-image-classpath", ".txt", keepFile = true) { file => IO.write(file, s"--class-path $classpath") Seq(s"@${file.absolutePath}") - } - else Seq("--class-path", classpath) + } else Seq("--class-path", classpath) cpArgs ++ Seq(s"-H:Name=$binaryName") ++ extraOptions ++ Seq(className) } Seq(nativeImageCommand) ++ nativeImageArguments @@ -125,26 +125,25 @@ object GraalVMNativeImagePlugin extends AutoPlugin { } } - private def buildInDockerContainer( - targetDirectory: File, - binaryName: String, - className: String, - classpathJars: Seq[(File, String)], - extraOptions: Seq[String], - dockerCommand: Seq[String], - resources: Seq[(File, String)], - image: String, - streams: TaskStreams - ): File = { + private def buildInDockerContainer(targetDirectory: File, + binaryName: String, + className: String, + classpathJars: Seq[(File, String)], + extraOptions: Seq[String], + platformArch: Option[String], + dockerCommand: Seq[String], + resources: Seq[(File, String)], + image: String, + streams: TaskStreams): File = { stage(targetDirectory, classpathJars, resources, streams) val graalDestDir = "/opt/graalvm" val stageDestDir = s"$graalDestDir/stage" val resourcesDestDir = s"$stageDestDir/resources" + val platformFlag = platformArch.map(arch => Seq("--platform", arch)).getOrElse(Seq.empty) - val command = dockerCommand ++ Seq( - "run", + val command = dockerCommand ++ Seq("run") ++ platformFlag ++ Seq( "--workdir", "/opt/graalvm", "--rm", @@ -166,15 +165,17 @@ object GraalVMNativeImagePlugin extends AutoPlugin { * This can be used to build a custom build image starting from a custom base image. Can be used like so: * * ``` - * (containerBuildImage in GraalVMNativeImage) := generateContainerBuildImage("my-docker-hub-username/my-graalvm").value + * (containerBuildImage in GraalVMNativeImage) := generateContainerBuildImage("my-docker-hub-username/my-graalvm", Some("arm64")).value * ``` * * The passed in docker image must have GraalVM installed and on the PATH, including the gu utility. */ - def generateContainerBuildImage(baseImage: String): Def.Initialize[Task[Option[String]]] = + def generateContainerBuildImage(baseImage: String, + platformArch: Option[String] = None): Def.Initialize[Task[Option[String]]] = Def.task { val dockerCommand = (DockerPlugin.autoImport.dockerExecCommand in GraalVMNativeImage).value val streams = Keys.streams.value + val platformValue = platformArch.getOrElse("local") val (baseName, tag) = baseImage.split(":", 2) match { case Array(n, t) => (n, t) @@ -182,8 +183,17 @@ object GraalVMNativeImagePlugin extends AutoPlugin { } val imageName = s"${baseName.replace('/', '-')}-native-image:$tag" + import sys.process._ - if ((dockerCommand ++ Seq("image", "ls", imageName, "--quiet")).!!.trim.isEmpty) { + val buildContainerExists = (dockerCommand ++ Seq( + "image", + "ls", + "--filter", + s"label=com.typesafe.sbt.packager.graalvmnativeimage.platform=$platformValue", + "--quiet", + imageName + )).!!.trim.nonEmpty + if (!buildContainerExists) { streams.log.info(s"Generating new GraalVM native-image image based on $baseImage: $imageName") val dockerContent = Dockerfile( @@ -194,7 +204,17 @@ object GraalVMNativeImagePlugin extends AutoPlugin { ExecCmd("ENTRYPOINT", "native-image") ).makeContent - val command = dockerCommand ++ Seq("build", "-t", imageName, "-") + val command = dockerCommand ++ Seq( + "buildx", + "build", + "--platform", + platformValue, + "--label", + s"com.typesafe.sbt.packager.graalvmnativeimage.platform=$platformValue", + "-t", + imageName, + "-" + ) val ret = sys.process.Process(command) #< new ByteArrayInputStream(dockerContent.getBytes()) ! @@ -209,12 +229,10 @@ object GraalVMNativeImagePlugin extends AutoPlugin { Some(imageName) } - private def stage( - targetDirectory: File, - classpathJars: Seq[(File, String)], - resources: Seq[(File, String)], - streams: TaskStreams - ): File = { + private def stage(targetDirectory: File, + classpathJars: Seq[(File, String)], + resources: Seq[(File, String)], + streams: TaskStreams): File = { val stageDir = targetDirectory / "stage" val mappings = classpathJars ++ resources.map { case (resource, path) => resource -> s"resources/$path" diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt new file mode 100644 index 000000000..2632d49b1 --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt @@ -0,0 +1,7 @@ +enablePlugins(GraalVMNativeImagePlugin) + +name := "docker-test" +version := "0.1.0" +graalVMNativeImageOptions := Seq("--no-fallback") +graalVMNativeImageGraalVersion := Some("22.3.2") +graalVMNativeImagePlatformArch := Some("arm64") diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/project/plugins.sbt b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/project/plugins.sbt new file mode 100644 index 000000000..218f1a27d --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % sys.props("project.version")) diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/src/main/scala/Main.scala b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/src/main/scala/Main.scala new file mode 100644 index 000000000..43c2459f3 --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/src/main/scala/Main.scala @@ -0,0 +1,5 @@ +object Main { + def main(args: Array[String]): Unit = { + println("Hello Graal") + } +} diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/test b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/test new file mode 100644 index 000000000..b6e174631 --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/test @@ -0,0 +1,3 @@ +# Generate the GraalVM native image +> show graalvm-native-image:packageBin +$ exec bash -c 'docker run --platform arm64 -v .:/test -w /test ubuntu ./target/graalvm-native-image/docker-test | grep -q "Hello Graal"' diff --git a/src/sbt-test/graalvm-native-image/docker-native-image/build.sbt b/src/sbt-test/graalvm-native-image/docker-native-image/build.sbt index 899d15a56..4b8d67ac7 100644 --- a/src/sbt-test/graalvm-native-image/docker-native-image/build.sbt +++ b/src/sbt-test/graalvm-native-image/docker-native-image/build.sbt @@ -2,4 +2,5 @@ enablePlugins(GraalVMNativeImagePlugin) name := "docker-test" version := "0.1.0" -graalVMNativeImageGraalVersion := Some("22.0.0.2") \ No newline at end of file +graalVMNativeImageOptions := Seq("--no-fallback") +graalVMNativeImageGraalVersion := Some("22.3.2") diff --git a/src/sbt-test/graalvm-native-image/simple-native-image/build.sbt b/src/sbt-test/graalvm-native-image/simple-native-image/build.sbt index 27042d757..b3eb05a63 100644 --- a/src/sbt-test/graalvm-native-image/simple-native-image/build.sbt +++ b/src/sbt-test/graalvm-native-image/simple-native-image/build.sbt @@ -1,5 +1,5 @@ enablePlugins(GraalVMNativeImagePlugin) name := "simple-test" - version := "0.1.0" +graalVMNativeImageOptions := Seq("--no-fallback") From a6c701e1a9dc477135e7d9431a3f54f84ec8842d Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Tue, 25 Jul 2023 04:10:59 +0900 Subject: [PATCH 02/17] scalafmt --- .../GraalVMNativeImagePlugin.scala | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala index e65b71c25..7057b0eb1 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala @@ -97,13 +97,15 @@ object GraalVMNativeImagePlugin extends AutoPlugin { } ) - private def buildLocal(targetDirectory: File, - binaryName: String, - nativeImageCommand: String, - className: String, - classpathJars: Seq[File], - extraOptions: Seq[String], - log: ProcessLogger): File = { + private def buildLocal( + targetDirectory: File, + binaryName: String, + nativeImageCommand: String, + className: String, + classpathJars: Seq[File], + extraOptions: Seq[String], + log: ProcessLogger + ): File = { targetDirectory.mkdirs() val command = { val nativeImageArguments = { @@ -113,7 +115,8 @@ object GraalVMNativeImagePlugin extends AutoPlugin { IO.withTemporaryFile("native-image-classpath", ".txt", keepFile = true) { file => IO.write(file, s"--class-path $classpath") Seq(s"@${file.absolutePath}") - } else Seq("--class-path", classpath) + } + else Seq("--class-path", classpath) cpArgs ++ Seq(s"-H:Name=$binaryName") ++ extraOptions ++ Seq(className) } Seq(nativeImageCommand) ++ nativeImageArguments @@ -125,16 +128,18 @@ object GraalVMNativeImagePlugin extends AutoPlugin { } } - private def buildInDockerContainer(targetDirectory: File, - binaryName: String, - className: String, - classpathJars: Seq[(File, String)], - extraOptions: Seq[String], - platformArch: Option[String], - dockerCommand: Seq[String], - resources: Seq[(File, String)], - image: String, - streams: TaskStreams): File = { + private def buildInDockerContainer( + targetDirectory: File, + binaryName: String, + className: String, + classpathJars: Seq[(File, String)], + extraOptions: Seq[String], + platformArch: Option[String], + dockerCommand: Seq[String], + resources: Seq[(File, String)], + image: String, + streams: TaskStreams + ): File = { stage(targetDirectory, classpathJars, resources, streams) @@ -170,8 +175,10 @@ object GraalVMNativeImagePlugin extends AutoPlugin { * * The passed in docker image must have GraalVM installed and on the PATH, including the gu utility. */ - def generateContainerBuildImage(baseImage: String, - platformArch: Option[String] = None): Def.Initialize[Task[Option[String]]] = + def generateContainerBuildImage( + baseImage: String, + platformArch: Option[String] = None + ): Def.Initialize[Task[Option[String]]] = Def.task { val dockerCommand = (DockerPlugin.autoImport.dockerExecCommand in GraalVMNativeImage).value val streams = Keys.streams.value @@ -229,10 +236,12 @@ object GraalVMNativeImagePlugin extends AutoPlugin { Some(imageName) } - private def stage(targetDirectory: File, - classpathJars: Seq[(File, String)], - resources: Seq[(File, String)], - streams: TaskStreams): File = { + private def stage( + targetDirectory: File, + classpathJars: Seq[(File, String)], + resources: Seq[(File, String)], + streams: TaskStreams + ): File = { val stageDir = targetDirectory / "stage" val mappings = classpathJars ++ resources.map { case (resource, path) => resource -> s"resources/$path" From 47b847c75b00997a6e8625c30c614e3b6e2c7b2a Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Tue, 25 Jul 2023 15:57:15 +0900 Subject: [PATCH 03/17] Update to use graalvm/setup-graalvm --- .github/workflows/validate-pr.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml index c70d8b646..e139fe740 100644 --- a/.github/workflows/validate-pr.yml +++ b/.github/workflows/validate-pr.yml @@ -252,11 +252,11 @@ jobs: restore-keys: | ${{ runner.os }}-ivy- - name: Setup GraalVM environment - uses: olafurpg/setup-scala@v10 + uses: graalvm/setup-graalvm@v1 with: - java-version: graalvm@22.3.2=tgz+https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-22.3.2/graalvm-ce-java17-linux-amd64-22.3.2.tar.gz - - name: Install native-image - run: gu install native-image + java-version: 17 + distribution: 'graalvm' + cache: 'sbt' - name: Validate run: sbt "^validateGraalVMNativeImage" From d352378f5f76daf86ec983c00cbb43606aadc434 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Wed, 26 Jul 2023 14:10:38 +0900 Subject: [PATCH 04/17] Update docs --- src/sphinx/formats/graalvm-native-image.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/sphinx/formats/graalvm-native-image.rst b/src/sphinx/formats/graalvm-native-image.rst index bd110785d..7c75d3603 100644 --- a/src/sphinx/formats/graalvm-native-image.rst +++ b/src/sphinx/formats/graalvm-native-image.rst @@ -76,6 +76,18 @@ customized using the following settings. graalVMNativeImageGraalVersion := Some("19.1.1") + ``graalVMNativeImagePlatformArch`` + Setting this enables building the native image on a different platform architecture. Requires ``graalVMNativeImageGraalVersion`` + to be set. Multiplatform builds is not supported. Defaults to the platform of the host. + If ``containerBuildImage`` is specified, ensure that your specified image has the same platform that you are targeting. + + For example: + + .. code-block:: scala + + graalVMNativeImagePlatformArch := Some("arm64") + graalVMNativeImagePlatformArch := Some("amd64") + ``containerBuildImage`` Explicitly set a build image to use. The image must execute the Graal ``native-image`` command as its entry point. From ebcdb21abf68d5c1c010f36c83c8d5e225eae6f6 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Thu, 27 Jul 2023 19:18:08 +0900 Subject: [PATCH 05/17] Support new graalvm container versions, backwards support for legacy versions --- .github/workflows/validate-pr.yml | 2 +- .../GraalVMNativeImagePlugin.scala | 38 ++++++++++++++----- .../docker-native-image-arm64/build.sbt | 2 +- .../docker-native-image-arm64/test | 2 +- .../build.sbt | 6 +++ .../project/plugins.sbt | 1 + .../src/main/scala/Main.scala | 5 +++ .../test | 3 ++ .../docker-native-image-legacy/build.sbt | 6 +++ .../project/plugins.sbt | 1 + .../src/main/scala/Main.scala | 5 +++ .../docker-native-image-legacy/test | 3 ++ .../docker-native-image/build.sbt | 2 +- src/sphinx/formats/graalvm-native-image.rst | 4 +- 14 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/build.sbt create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/project/plugins.sbt create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/src/main/scala/Main.scala create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/test create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-legacy/build.sbt create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-legacy/project/plugins.sbt create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-legacy/src/main/scala/Main.scala create mode 100644 src/sbt-test/graalvm-native-image/docker-native-image-legacy/test diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml index e139fe740..ef2f2c2e7 100644 --- a/.github/workflows/validate-pr.yml +++ b/.github/workflows/validate-pr.yml @@ -254,7 +254,7 @@ jobs: - name: Setup GraalVM environment uses: graalvm/setup-graalvm@v1 with: - java-version: 17 + java-version: 17.0.8 distribution: 'graalvm' cache: 'sbt' - name: Validate diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala index 7057b0eb1..5e2a6d376 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala @@ -27,7 +27,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { import autoImport._ - private val GraalVMBaseImage = "ghcr.io/graalvm/graalvm-ce" + private val GraalVMBaseImagePath = "ghcr.io/graalvm/" override def requires: Plugins = JavaAppPackaging @@ -48,9 +48,20 @@ object GraalVMNativeImagePlugin extends AutoPlugin { includeFilter := "*", resources := resourceDirectories.value.descendantsExcept(includeFilter.value, excludeFilter.value).get, UniversalPlugin.autoImport.containerBuildImage := Def.taskDyn { + val splitPackageVersion = "(.*):(.*)".r graalVMNativeImageGraalVersion.value match { - case Some(tag) => generateContainerBuildImage(s"$GraalVMBaseImage:$tag", graalVMNativeImagePlatformArch.value) - case None => Def.task(None: Option[String]) + case Some(splitPackageVersion(packageName, tag)) => + packageName match { + case "graalvm-community" => Def.task(Some(s"$GraalVMBaseImagePath$packageName:$tag"): Option[String]) + case _ => + generateContainerBuildImage( + s"${GraalVMBaseImagePath}graalvm-ce:$tag", + graalVMNativeImagePlatformArch.value + ) + } + case Some(tag) => + generateContainerBuildImage(s"${GraalVMBaseImagePath}graalvm-ce:$tag", graalVMNativeImagePlatformArch.value) + case None => Def.task(None: Option[String]) } }.value, packageBin := { @@ -140,21 +151,26 @@ object GraalVMNativeImagePlugin extends AutoPlugin { image: String, streams: TaskStreams ): File = { - + import sys.process._ stage(targetDirectory, classpathJars, resources, streams) val graalDestDir = "/opt/graalvm" val stageDestDir = s"$graalDestDir/stage" val resourcesDestDir = s"$stageDestDir/resources" - val platformFlag = platformArch.map(arch => Seq("--platform", arch)).getOrElse(Seq.empty) + val hostPlatform = + (dockerCommand ++ Seq("system", "info", "--format", "{{.OSType}}/{{.Architecture}}")).!!.trim - val command = dockerCommand ++ Seq("run") ++ platformFlag ++ Seq( + val command = dockerCommand ++ Seq( + "run", "--workdir", "/opt/graalvm", "--rm", + "--platform", + platformArch.getOrElse(hostPlatform), "-v", s"${targetDirectory.getAbsolutePath}:$graalDestDir", image, + "native-image", "-cp", (resourcesDestDir +: classpathJars.map(jar => s"$stageDestDir/" + jar._2)).mkString(":"), s"-H:Name=$binaryName" @@ -180,9 +196,13 @@ object GraalVMNativeImagePlugin extends AutoPlugin { platformArch: Option[String] = None ): Def.Initialize[Task[Option[String]]] = Def.task { + import sys.process._ + val dockerCommand = (DockerPlugin.autoImport.dockerExecCommand in GraalVMNativeImage).value val streams = Keys.streams.value - val platformValue = platformArch.getOrElse("local") + val hostPlatform = + (dockerCommand ++ Seq("system", "info", "--format", "{{.OSType}}/{{.Architecture}}")).!!.trim + val platformValue = platformArch.getOrElse(hostPlatform) val (baseName, tag) = baseImage.split(":", 2) match { case Array(n, t) => (n, t) @@ -208,7 +228,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { Cmd("WORKDIR", "/opt/graalvm"), ExecCmd("RUN", "gu", "install", "native-image"), ExecCmd("RUN", "sh", "-c", "ln -s /opt/graalvm-ce-*/bin/native-image /usr/local/bin/native-image"), - ExecCmd("ENTRYPOINT", "native-image") + ExecCmd("CMD", "native-image") ).makeContent val command = dockerCommand ++ Seq( @@ -246,7 +266,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { val mappings = classpathJars ++ resources.map { case (resource, path) => resource -> s"resources/$path" } - Stager.stage(GraalVMBaseImage)(streams, stageDir, mappings) + Stager.stage(GraalVMBaseImagePath)(streams, stageDir, mappings) } } diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt index 2632d49b1..8a71e3400 100644 --- a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt +++ b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt @@ -3,5 +3,5 @@ enablePlugins(GraalVMNativeImagePlugin) name := "docker-test" version := "0.1.0" graalVMNativeImageOptions := Seq("--no-fallback") -graalVMNativeImageGraalVersion := Some("22.3.2") +graalVMNativeImageGraalVersion := Some("graalvm-community:17.0.8") graalVMNativeImagePlatformArch := Some("arm64") diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/test b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/test index b6e174631..f4e58abe3 100644 --- a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/test +++ b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/test @@ -1,3 +1,3 @@ # Generate the GraalVM native image > show graalvm-native-image:packageBin -$ exec bash -c 'docker run --platform arm64 -v .:/test -w /test ubuntu ./target/graalvm-native-image/docker-test | grep -q "Hello Graal"' +$ exec bash -c 'docker run --rm --platform arm64 -v .:/test -w /test ubuntu ./target/graalvm-native-image/docker-test | grep -q "Hello Graal"' diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/build.sbt b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/build.sbt new file mode 100644 index 000000000..2f60ef3e7 --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/build.sbt @@ -0,0 +1,6 @@ +enablePlugins(GraalVMNativeImagePlugin) + +name := "docker-test" +version := "0.1.0" +graalVMNativeImageOptions := Seq("--no-fallback") +graalVMNativeImageGraalVersion := Some("graalvm-ce:22.3.3") diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/project/plugins.sbt b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/project/plugins.sbt new file mode 100644 index 000000000..218f1a27d --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % sys.props("project.version")) diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/src/main/scala/Main.scala b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/src/main/scala/Main.scala new file mode 100644 index 000000000..43c2459f3 --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/src/main/scala/Main.scala @@ -0,0 +1,5 @@ +object Main { + def main(args: Array[String]): Unit = { + println("Hello Graal") + } +} diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/test b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/test new file mode 100644 index 000000000..3d6230bcd --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/test @@ -0,0 +1,3 @@ +# Generate the GraalVM native image +> show graalvm-native-image:packageBin +$ exec bash -c 'target/graalvm-native-image/docker-test | grep -q "Hello Graal"' diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-legacy/build.sbt b/src/sbt-test/graalvm-native-image/docker-native-image-legacy/build.sbt new file mode 100644 index 000000000..c82125a5b --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-legacy/build.sbt @@ -0,0 +1,6 @@ +enablePlugins(GraalVMNativeImagePlugin) + +name := "docker-test" +version := "0.1.0" +graalVMNativeImageOptions := Seq("--no-fallback") +graalVMNativeImageGraalVersion := Some("22.3.3") diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-legacy/project/plugins.sbt b/src/sbt-test/graalvm-native-image/docker-native-image-legacy/project/plugins.sbt new file mode 100644 index 000000000..218f1a27d --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-legacy/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % sys.props("project.version")) diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-legacy/src/main/scala/Main.scala b/src/sbt-test/graalvm-native-image/docker-native-image-legacy/src/main/scala/Main.scala new file mode 100644 index 000000000..43c2459f3 --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-legacy/src/main/scala/Main.scala @@ -0,0 +1,5 @@ +object Main { + def main(args: Array[String]): Unit = { + println("Hello Graal") + } +} diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-legacy/test b/src/sbt-test/graalvm-native-image/docker-native-image-legacy/test new file mode 100644 index 000000000..3d6230bcd --- /dev/null +++ b/src/sbt-test/graalvm-native-image/docker-native-image-legacy/test @@ -0,0 +1,3 @@ +# Generate the GraalVM native image +> show graalvm-native-image:packageBin +$ exec bash -c 'target/graalvm-native-image/docker-test | grep -q "Hello Graal"' diff --git a/src/sbt-test/graalvm-native-image/docker-native-image/build.sbt b/src/sbt-test/graalvm-native-image/docker-native-image/build.sbt index 4b8d67ac7..9663b1fa2 100644 --- a/src/sbt-test/graalvm-native-image/docker-native-image/build.sbt +++ b/src/sbt-test/graalvm-native-image/docker-native-image/build.sbt @@ -3,4 +3,4 @@ enablePlugins(GraalVMNativeImagePlugin) name := "docker-test" version := "0.1.0" graalVMNativeImageOptions := Seq("--no-fallback") -graalVMNativeImageGraalVersion := Some("22.3.2") +graalVMNativeImageGraalVersion := Some("graalvm-community:17.0.8") diff --git a/src/sphinx/formats/graalvm-native-image.rst b/src/sphinx/formats/graalvm-native-image.rst index 7c75d3603..3ae660917 100644 --- a/src/sphinx/formats/graalvm-native-image.rst +++ b/src/sphinx/formats/graalvm-native-image.rst @@ -74,7 +74,9 @@ customized using the following settings. .. code-block:: scala - graalVMNativeImageGraalVersion := Some("19.1.1") + graalVMNativeImageGraalVersion := Some("19.1.1") // Legacy GraalVM versions supported up to 22.3.3 + graalVMNativeImageGraalVersion := Some("graalvm-ce:19.1.1") // Legacy GraalVM versions supported up to 22.3.3 + graalVMNativeImageGraalVersion := Some("graalvm-community:17.0.8") // New GraalVM version scheme ``graalVMNativeImagePlatformArch`` Setting this enables building the native image on a different platform architecture. Requires ``graalVMNativeImageGraalVersion`` From 4e45cc09b840fb79950a4bed88a44b0d78a96c41 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Tue, 1 Aug 2023 04:29:33 +0900 Subject: [PATCH 06/17] Add support for native-image images, revert to use entrypoint --- .../graalvmnativeimage/GraalVMNativeImagePlugin.scala | 11 ++++++----- .../docker-native-image-arm64/build.sbt | 2 +- .../build.sbt | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala index 5e2a6d376..a4916bcfc 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala @@ -52,12 +52,14 @@ object GraalVMNativeImagePlugin extends AutoPlugin { graalVMNativeImageGraalVersion.value match { case Some(splitPackageVersion(packageName, tag)) => packageName match { - case "graalvm-community" => Def.task(Some(s"$GraalVMBaseImagePath$packageName:$tag"): Option[String]) - case _ => + case "native-image-community" | "native-image" => + Def.task(Some(s"$GraalVMBaseImagePath$packageName:$tag"): Option[String]) + case "graalvm-community" | "graalvm-ce" => generateContainerBuildImage( - s"${GraalVMBaseImagePath}graalvm-ce:$tag", + s"${GraalVMBaseImagePath}$packageName:$tag", graalVMNativeImagePlatformArch.value ) + case _ => sys.error("Other ghcr.io/graalvm images are unsupported") } case Some(tag) => generateContainerBuildImage(s"${GraalVMBaseImagePath}graalvm-ce:$tag", graalVMNativeImagePlatformArch.value) @@ -170,7 +172,6 @@ object GraalVMNativeImagePlugin extends AutoPlugin { "-v", s"${targetDirectory.getAbsolutePath}:$graalDestDir", image, - "native-image", "-cp", (resourcesDestDir +: classpathJars.map(jar => s"$stageDestDir/" + jar._2)).mkString(":"), s"-H:Name=$binaryName" @@ -228,7 +229,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { Cmd("WORKDIR", "/opt/graalvm"), ExecCmd("RUN", "gu", "install", "native-image"), ExecCmd("RUN", "sh", "-c", "ln -s /opt/graalvm-ce-*/bin/native-image /usr/local/bin/native-image"), - ExecCmd("CMD", "native-image") + ExecCmd("ENTRYPOINT", "native-image") ).makeContent val command = dockerCommand ++ Seq( diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt index 8a71e3400..bd23343a2 100644 --- a/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt +++ b/src/sbt-test/graalvm-native-image/docker-native-image-arm64/build.sbt @@ -3,5 +3,5 @@ enablePlugins(GraalVMNativeImagePlugin) name := "docker-test" version := "0.1.0" graalVMNativeImageOptions := Seq("--no-fallback") -graalVMNativeImageGraalVersion := Some("graalvm-community:17.0.8") +graalVMNativeImageGraalVersion := Some("native-image-community:17.0.8") graalVMNativeImagePlatformArch := Some("arm64") diff --git a/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/build.sbt b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/build.sbt index 2f60ef3e7..38b1dd18c 100644 --- a/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/build.sbt +++ b/src/sbt-test/graalvm-native-image/docker-native-image-legacy-specify-package/build.sbt @@ -3,4 +3,4 @@ enablePlugins(GraalVMNativeImagePlugin) name := "docker-test" version := "0.1.0" graalVMNativeImageOptions := Seq("--no-fallback") -graalVMNativeImageGraalVersion := Some("graalvm-ce:22.3.3") +graalVMNativeImageGraalVersion := Some("native-image:22.3.3") From 46d044d96de82afd35124523edad58f60525fc20 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Tue, 1 Aug 2023 04:39:03 +0900 Subject: [PATCH 07/17] Update docs --- src/sphinx/formats/graalvm-native-image.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sphinx/formats/graalvm-native-image.rst b/src/sphinx/formats/graalvm-native-image.rst index 3ae660917..80f3f0971 100644 --- a/src/sphinx/formats/graalvm-native-image.rst +++ b/src/sphinx/formats/graalvm-native-image.rst @@ -76,11 +76,13 @@ customized using the following settings. graalVMNativeImageGraalVersion := Some("19.1.1") // Legacy GraalVM versions supported up to 22.3.3 graalVMNativeImageGraalVersion := Some("graalvm-ce:19.1.1") // Legacy GraalVM versions supported up to 22.3.3 + graalVMNativeImageGraalVersion := Some("native-image:19.1.1") // Uses the legacy native-image image from GraalVM directly graalVMNativeImageGraalVersion := Some("graalvm-community:17.0.8") // New GraalVM version scheme + graalVMNativeImageGraalVersion := Some("native-image-community:17.0.8") // Uses the native-image image from GraalVM directly ``graalVMNativeImagePlatformArch`` Setting this enables building the native image on a different platform architecture. Requires ``graalVMNativeImageGraalVersion`` - to be set. Multiplatform builds is not supported. Defaults to the platform of the host. + or ``containerBuildImage`` to be set. Multiplatform builds is not supported. Defaults to the platform of the host. If ``containerBuildImage`` is specified, ensure that your specified image has the same platform that you are targeting. For example: From 8db749503029673d610f403e0535b9d05fe5a5b2 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sat, 5 Aug 2023 21:35:48 +0900 Subject: [PATCH 08/17] Attempt to fix binary compatibility --- .../graalvmnativeimage/GraalVMNativeImageKeys.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImageKeys.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImageKeys.scala index b3e2aa8ea..01264be6b 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImageKeys.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImageKeys.scala @@ -14,12 +14,12 @@ trait GraalVMNativeImageKeys { val graalVMNativeImageGraalVersion = settingKey[Option[String]]( "Version of GraalVM to build with. Setting this has the effect of generating a container build image to build the native image with this version of GraalVM." ) - - val graalVMNativeImagePlatformArch = settingKey[Option[String]]( - "Platform architecture of GraalVM to build with. This only works when building the native image with a container." - ) } trait GraalVMNativeImageKeysEx extends GraalVMNativeImageKeys { val graalVMNativeImageCommand = settingKey[String]("GraalVM native-image executable command") + + val graalVMNativeImagePlatformArch = settingKey[Option[String]]( + "Platform architecture of GraalVM to build with. This only works when building the native image with a container." + ) } From 5928aa8473fb654e97f0cf4382ce40d993debc64 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sat, 5 Aug 2023 21:47:44 +0900 Subject: [PATCH 09/17] sbt cross build version attempt --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 4cab8248e..8556db2cd 100644 --- a/build.sbt +++ b/build.sbt @@ -6,7 +6,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / scalaVersion := "2.12.13" // crossBuildingSettings -crossSbtVersions := Vector("1.1.6") +crossSbtVersions := Vector("1.9.3") Compile / scalacOptions ++= Seq("-deprecation") javacOptions ++= Seq("-source", "1.8", "-target", "1.8") From b4fa76d9e7e646db566ebb404cc89d2bedfab617 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sat, 5 Aug 2023 23:28:00 +0900 Subject: [PATCH 10/17] Add docker buildx to github action --- .github/workflows/validate-pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml index ef2f2c2e7..e0b0a4d2a 100644 --- a/.github/workflows/validate-pr.yml +++ b/.github/workflows/validate-pr.yml @@ -257,6 +257,8 @@ jobs: java-version: 17.0.8 distribution: 'graalvm' cache: 'sbt' + - name: Docker Setup Buildx + uses: docker/setup-buildx-action@v2.9.1 - name: Validate run: sbt "^validateGraalVMNativeImage" From 16fea55815118463180b9007879e68c5eed0fc84 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sat, 5 Aug 2023 23:43:55 +0900 Subject: [PATCH 11/17] Fix architecture label --- .../GraalVMNativeImagePlugin.scala | 58 ++++++++----------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala index a4916bcfc..954141109 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala @@ -110,15 +110,13 @@ object GraalVMNativeImagePlugin extends AutoPlugin { } ) - private def buildLocal( - targetDirectory: File, - binaryName: String, - nativeImageCommand: String, - className: String, - classpathJars: Seq[File], - extraOptions: Seq[String], - log: ProcessLogger - ): File = { + private def buildLocal(targetDirectory: File, + binaryName: String, + nativeImageCommand: String, + className: String, + classpathJars: Seq[File], + extraOptions: Seq[String], + log: ProcessLogger): File = { targetDirectory.mkdirs() val command = { val nativeImageArguments = { @@ -128,8 +126,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { IO.withTemporaryFile("native-image-classpath", ".txt", keepFile = true) { file => IO.write(file, s"--class-path $classpath") Seq(s"@${file.absolutePath}") - } - else Seq("--class-path", classpath) + } else Seq("--class-path", classpath) cpArgs ++ Seq(s"-H:Name=$binaryName") ++ extraOptions ++ Seq(className) } Seq(nativeImageCommand) ++ nativeImageArguments @@ -141,18 +138,16 @@ object GraalVMNativeImagePlugin extends AutoPlugin { } } - private def buildInDockerContainer( - targetDirectory: File, - binaryName: String, - className: String, - classpathJars: Seq[(File, String)], - extraOptions: Seq[String], - platformArch: Option[String], - dockerCommand: Seq[String], - resources: Seq[(File, String)], - image: String, - streams: TaskStreams - ): File = { + private def buildInDockerContainer(targetDirectory: File, + binaryName: String, + className: String, + classpathJars: Seq[(File, String)], + extraOptions: Seq[String], + platformArch: Option[String], + dockerCommand: Seq[String], + resources: Seq[(File, String)], + image: String, + streams: TaskStreams): File = { import sys.process._ stage(targetDirectory, classpathJars, resources, streams) @@ -161,6 +156,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { val resourcesDestDir = s"$stageDestDir/resources" val hostPlatform = (dockerCommand ++ Seq("system", "info", "--format", "{{.OSType}}/{{.Architecture}}")).!!.trim + .replace("x86_64", "amd64") val command = dockerCommand ++ Seq( "run", @@ -192,10 +188,8 @@ object GraalVMNativeImagePlugin extends AutoPlugin { * * The passed in docker image must have GraalVM installed and on the PATH, including the gu utility. */ - def generateContainerBuildImage( - baseImage: String, - platformArch: Option[String] = None - ): Def.Initialize[Task[Option[String]]] = + def generateContainerBuildImage(baseImage: String, + platformArch: Option[String] = None): Def.Initialize[Task[Option[String]]] = Def.task { import sys.process._ @@ -257,12 +251,10 @@ object GraalVMNativeImagePlugin extends AutoPlugin { Some(imageName) } - private def stage( - targetDirectory: File, - classpathJars: Seq[(File, String)], - resources: Seq[(File, String)], - streams: TaskStreams - ): File = { + private def stage(targetDirectory: File, + classpathJars: Seq[(File, String)], + resources: Seq[(File, String)], + streams: TaskStreams): File = { val stageDir = targetDirectory / "stage" val mappings = classpathJars ++ resources.map { case (resource, path) => resource -> s"resources/$path" From b48618f0a95a7be6169ac6bf7b1bb24483a2a918 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sat, 5 Aug 2023 23:51:16 +0900 Subject: [PATCH 12/17] Ignore bsp --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 96a52da37..a0d11fa9a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ target/ .ensime* .bloop/* .metals/* + +.bsp/ From bb0faad9d4058dd1eb95a72d41d4bce1717f617d Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sat, 5 Aug 2023 23:52:11 +0900 Subject: [PATCH 13/17] fmt --- .../GraalVMNativeImagePlugin.scala | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala index 954141109..fd1fa63db 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala @@ -110,13 +110,15 @@ object GraalVMNativeImagePlugin extends AutoPlugin { } ) - private def buildLocal(targetDirectory: File, - binaryName: String, - nativeImageCommand: String, - className: String, - classpathJars: Seq[File], - extraOptions: Seq[String], - log: ProcessLogger): File = { + private def buildLocal( + targetDirectory: File, + binaryName: String, + nativeImageCommand: String, + className: String, + classpathJars: Seq[File], + extraOptions: Seq[String], + log: ProcessLogger + ): File = { targetDirectory.mkdirs() val command = { val nativeImageArguments = { @@ -126,7 +128,8 @@ object GraalVMNativeImagePlugin extends AutoPlugin { IO.withTemporaryFile("native-image-classpath", ".txt", keepFile = true) { file => IO.write(file, s"--class-path $classpath") Seq(s"@${file.absolutePath}") - } else Seq("--class-path", classpath) + } + else Seq("--class-path", classpath) cpArgs ++ Seq(s"-H:Name=$binaryName") ++ extraOptions ++ Seq(className) } Seq(nativeImageCommand) ++ nativeImageArguments @@ -138,16 +141,18 @@ object GraalVMNativeImagePlugin extends AutoPlugin { } } - private def buildInDockerContainer(targetDirectory: File, - binaryName: String, - className: String, - classpathJars: Seq[(File, String)], - extraOptions: Seq[String], - platformArch: Option[String], - dockerCommand: Seq[String], - resources: Seq[(File, String)], - image: String, - streams: TaskStreams): File = { + private def buildInDockerContainer( + targetDirectory: File, + binaryName: String, + className: String, + classpathJars: Seq[(File, String)], + extraOptions: Seq[String], + platformArch: Option[String], + dockerCommand: Seq[String], + resources: Seq[(File, String)], + image: String, + streams: TaskStreams + ): File = { import sys.process._ stage(targetDirectory, classpathJars, resources, streams) @@ -188,8 +193,10 @@ object GraalVMNativeImagePlugin extends AutoPlugin { * * The passed in docker image must have GraalVM installed and on the PATH, including the gu utility. */ - def generateContainerBuildImage(baseImage: String, - platformArch: Option[String] = None): Def.Initialize[Task[Option[String]]] = + def generateContainerBuildImage( + baseImage: String, + platformArch: Option[String] = None + ): Def.Initialize[Task[Option[String]]] = Def.task { import sys.process._ @@ -251,10 +258,12 @@ object GraalVMNativeImagePlugin extends AutoPlugin { Some(imageName) } - private def stage(targetDirectory: File, - classpathJars: Seq[(File, String)], - resources: Seq[(File, String)], - streams: TaskStreams): File = { + private def stage( + targetDirectory: File, + classpathJars: Seq[(File, String)], + resources: Seq[(File, String)], + streams: TaskStreams + ): File = { val stageDir = targetDirectory / "stage" val mappings = classpathJars ++ resources.map { case (resource, path) => resource -> s"resources/$path" From 3161c3cc479831e45cc33c6972171b1a83719eca Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sun, 6 Aug 2023 00:04:24 +0900 Subject: [PATCH 14/17] Fix buildx docker image not found --- .../packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala index fd1fa63db..fdabc8d3a 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala @@ -238,6 +238,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { "build", "--platform", platformValue, + "--load", "--label", s"com.typesafe.sbt.packager.graalvmnativeimage.platform=$platformValue", "-t", From 5cc6b9b7c96cbafe2ee50a4bc8dfb1727c0faa2e Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sun, 6 Aug 2023 00:19:55 +0900 Subject: [PATCH 15/17] Add QEMU to github action --- .github/workflows/validate-pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml index e0b0a4d2a..177e8380b 100644 --- a/.github/workflows/validate-pr.yml +++ b/.github/workflows/validate-pr.yml @@ -257,6 +257,8 @@ jobs: java-version: 17.0.8 distribution: 'graalvm' cache: 'sbt' + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Docker Setup Buildx uses: docker/setup-buildx-action@v2.9.1 - name: Validate From c19779508c126bf938e1cca6d4200ae1af8c5c6c Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sun, 6 Aug 2023 01:50:34 +0900 Subject: [PATCH 16/17] Retain docker build compat if not using graalVMNativeImagePlatformArch --- .../GraalVMNativeImagePlugin.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala index fdabc8d3a..016b4e6ff 100644 --- a/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala +++ b/src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala @@ -204,6 +204,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { val streams = Keys.streams.value val hostPlatform = (dockerCommand ++ Seq("system", "info", "--format", "{{.OSType}}/{{.Architecture}}")).!!.trim + .replace("x86_64", "amd64") val platformValue = platformArch.getOrElse(hostPlatform) val (baseName, tag) = baseImage.split(":", 2) match { @@ -233,7 +234,16 @@ object GraalVMNativeImagePlugin extends AutoPlugin { ExecCmd("ENTRYPOINT", "native-image") ).makeContent - val command = dockerCommand ++ Seq( + val buildCommand = dockerCommand ++ Seq( + "build", + "--label", + s"com.typesafe.sbt.packager.graalvmnativeimage.platform=$platformValue", + "-t", + imageName, + "-" + ) + + val buildxCommand = dockerCommand ++ Seq( "buildx", "build", "--platform", @@ -246,7 +256,7 @@ object GraalVMNativeImagePlugin extends AutoPlugin { "-" ) - val ret = sys.process.Process(command) #< + val ret = sys.process.Process(platformArch.map(_ => buildxCommand).getOrElse(buildCommand)) #< new ByteArrayInputStream(dockerContent.getBytes()) ! DockerPlugin.publishLocalLogger(streams.log) From 453272053c63caed05511b2b82277eb0b0ddad89 Mon Sep 17 00:00:00 2001 From: Kingston Chan Date: Sun, 6 Aug 2023 17:41:38 +0900 Subject: [PATCH 17/17] Update documentation --- src/sphinx/formats/graalvm-native-image.rst | 30 ++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/sphinx/formats/graalvm-native-image.rst b/src/sphinx/formats/graalvm-native-image.rst index 80f3f0971..386943dd7 100644 --- a/src/sphinx/formats/graalvm-native-image.rst +++ b/src/sphinx/formats/graalvm-native-image.rst @@ -5,17 +5,20 @@ GraalVM Native Image Plugin GraalVM's ``native-image`` compiles Java programs AOT (ahead-of-time) into native binaries. - https://www.graalvm.org/22.1/reference-manual/native-image/ documents the AOT compilation of GraalVM. + https://www.graalvm.org/latest/reference-manual/native-image/ documents the AOT compilation of GraalVM. The plugin supports both using a local installation of the GraalVM ``native-image`` utility, or building inside a Docker container. If you intend to run the native image on Linux, then building inside a Docker container is recommended since GraalVM native images can only be built for the platform they are built on. By building in a Docker -container, you can build Linux native images not just on Linux but also on Windows and macOS. +container, you can build Linux native images not only on Linux but also on Windows and macOS and for different architectures +like amd64 or arm64. Requirements ------------ To build using a local installation of GraalVM, you must have the ``native-image`` utility of GraalVM in your ``PATH``. +To build using a docker container, you must have a working installation of docker. +To build for a different architecture, you must have docker with the buildx plugin and QEMU set up for the target architecture. ``native-image`` quick installation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -65,10 +68,16 @@ By default, a local build will be done, expecting the ``native-image`` command t customized using the following settings. ``graalVMNativeImageGraalVersion`` - Setting this enables generating a Docker container to build the native image, and then building it in that container. - It must correspond to a valid version of the - `Oracle GraalVM Community Edition Docker image `_. This setting has no - effect if ``containerBuildImage`` is explicitly set. + Setting this enables using a Docker container to build the native image. + It should be in the format ``:``. `Supported packages `_ include: + * ``graalvm-ce`` - Versions prior to and including 22.3.3. An intermediate image will be created. + * ``native-image`` - Versions prior to and including 22.3.3. The docker image will be used directly. + * ``graalvm-community`` - Versions after and including 17.0.7. An intermediate image will be created. + * ``native-image-community`` - Versions after and including 17.0.7. The docker image will be used directly. + + The legacy format of specifying the version number is supported up to 22.3.3 + + This setting has no effect if ``containerBuildImage`` is explicitly set. For example: @@ -81,16 +90,19 @@ customized using the following settings. graalVMNativeImageGraalVersion := Some("native-image-community:17.0.8") // Uses the native-image image from GraalVM directly ``graalVMNativeImagePlatformArch`` - Setting this enables building the native image on a different platform architecture. Requires ``graalVMNativeImageGraalVersion`` - or ``containerBuildImage`` to be set. Multiplatform builds is not supported. Defaults to the platform of the host. + Setting this enables building the native image for a different platform architecture. Requires ``graalVMNativeImageGraalVersion`` + or ``containerBuildImage`` to be set. Multiplatform builds are currently not supported. Defaults to the platform of the host. If ``containerBuildImage`` is specified, ensure that your specified image has the same platform that you are targeting. + Requires Docker buildx plugin with a valid builder and QEMU set up for the target platform architecture. + `See here for more information `_. + For example: .. code-block:: scala graalVMNativeImagePlatformArch := Some("arm64") - graalVMNativeImagePlatformArch := Some("amd64") + graalVMNativeImagePlatformArch := Some("linux/amd64") ``containerBuildImage``