fix: r-lang in ubuntu:22.04 (#1967)

Signed-off-by: Keming <[email protected]>

e2e/docs/rlang_mnist_test.go → e2e/docs/rlang_iris_test.go

@@ -21,17 +21,17 @@ import (
 	"github.com/tensorchord/envd/e2e"
 )
 
-var _ = Describe("rlang_mnist", Ordered, func() {
-	exampleName := "rlang_mnist"
+var _ = Describe("rlang_iris", Ordered, func() {
+	exampleName := "rlang_iris"
 	testcase := "e2e"
 	e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase)
 	BeforeAll(e.BuildImage(true))
 	BeforeEach(e.RunContainer())
 	It("execute runtime command `Rscript`", func() {
-		res, err := e.ExecRuntimeCommand("rlang-mnist")
+		res, err := e.ExecRuntimeCommand("rlang-iris")
 		Expect(err).To(BeNil())
-		isNumeric := "TRUE"
-		Expect(res).To(BeEquivalentTo(isNumeric))
+		Expect(res).To(ContainSubstring("classif.acc"))
+		Expect(res).To(ContainSubstring("classif.ce"))
 	})
 	AfterEach(e.DestroyContainer())
 })

e2e/docs/testdata/complex/build.envd

@@ -3,7 +3,7 @@
 
 def build():
     base(dev=True)
-    install.cuda(version="11.2.2", cudnn="8")
+    install.cuda(version="12.3.2", cudnn="9")
     install.conda()
     install.python()
     config.apt_source(

e2e/docs/testdata/julia_mnist/build.envd

@@ -1,5 +1,6 @@
 # syntax=v1
 
+
 def build():
     base(dev=True)
     install.julia()

e2e/docs/testdata/rlang/build.envd

@@ -4,4 +4,4 @@
 def build():
     base(dev=True)
     install.r_lang()
-    install.r_packages(name=["drat"])
+    install.r_packages(name=["remotes"])

e2e/docs/testdata/rlang_iris/build.envd

@@ -0,0 +1,9 @@
+# syntax=v1
+
+
+def build():
+    base(dev=True)
+    install.apt_packages(name=["build-essential"])
+    install.r_lang()
+    install.r_packages(name=["mlr3", "mlr3learners"])
+    runtime.command(commands={"rlang-iris": "Rscript iris.r 2> /dev/null"})

e2e/docs/testdata/rlang_iris/iris.r

@@ -0,0 +1,36 @@
+# Example from https://github.com/mlr-org/mlr3gallery
+# MIT License Copyright (c) 2019 mlr-org
+
+library(mlr3learners)
+
+# creates mlr3 task from scratch, from a data.frame
+# 'target' names the column in the dataset we want to learn to predict
+task = as_task_classif(iris, target = "Species")
+# in this case we could also take the iris example from mlr3's dictionary of shipped example tasks
+# 2 equivalent calls to create a task. The second is just sugar for the user.
+task = mlr_tasks$get("iris")
+task = tsk("iris")
+# print(task)
+# create learner from dictionary of mlr3learners
+# 2 equivalent calls:
+learner_1 = mlr_learners$get("classif.rpart")
+learner_1 = lrn("classif.rpart")
+# print(learner_1)
+
+# train learner on subset of task
+learner_1$train(task, row_ids = 1:120)
+# this is what the decision tree looks like
+# print(learner_1$model)
+# predict using observations from task
+prediction = learner_1$predict(task, row_ids = 121:150)
+# predict using "new" observations from an external data.frame
+prediction = learner_1$predict_newdata(newdata = iris[121:150, ])
+# print(prediction)
+
+# head(as.data.table(mlr_measures))
+scores = prediction$score(msr("classif.acc"))
+print(scores)
+scores = prediction$score(msrs(c("classif.acc", "classif.ce")))
+print(scores)
+cm = prediction$confusion
+print(cm)

pkg/lang/ir/v1/julia.go

@@ -36,7 +36,6 @@ var downloadJuliaBashScript string
 // getJuliaBinary returns the llb.State only after setting up Julia environment
 // A successful run of getJuliaBinary should set up the Julia environment
 func (g generalGraph) getJuliaBinary(root llb.State) llb.State {
-
 	base := llb.Image(builderImage)
 	builder := base.
 		Run(llb.Shlexf("sh -c '%s'", downloadJuliaBashScript),
@@ -57,33 +56,26 @@ func (g generalGraph) getJuliaBinary(root llb.State) llb.State {
 // installJulia returns the llb.State only after adding the Julia environment to $PATH
 // A successful run of installJulia should add Julia to global environment path
 func (g *generalGraph) installJulia(root llb.State) llb.State {
-
 	confJulia := g.getJuliaBinary(root)
 	confJulia = g.updateEnvPath(confJulia, juliaBinDir)
-
 	return confJulia
 }
 
 // installJuliaPackages returns the llb.State only after installing required Julia packages
 // A successful run of installJuliaPackages should install Julia packages under "/opt/julia/user_packages" and export the path
 func (g *generalGraph) installJuliaPackages(root llb.State) llb.State {
-
 	if len(g.JuliaPackages) == 0 {
 		return root
 	}
 
 	root = root.File(llb.Mkdir(juliaPkgDir, 0755, llb.WithParents(true)),
 		llb.WithCustomName("[internal] creating folder for julia packages"))
-
 	// Allow root to utilize the installed Julia environment
 	root = g.updateEnvPath(root, juliaBinDir)
-
 	// Export "/opt/julia/user_packages" as the additional library path for root
 	root = root.AddEnv("JULIA_DEPOT_PATH", juliaPkgDir)
-
 	// Export "/opt/julia/user_packages" as the additional library path for users
 	g.RuntimeEnviron["JULIA_DEPOT_PATH"] = juliaPkgDir
-
 	// Change owner of the "/opt/julia/user_packages" to users
 	g.UserDirectories = append(g.UserDirectories, juliaPkgDir)
 
@@ -93,6 +85,5 @@ func (g *generalGraph) installJuliaPackages(root llb.State) llb.State {
 			Run(llb.Shlex(command), llb.WithCustomNamef("[internal] installing Julia packages: %s", strings.Join(packages, " ")))
 		root = run.Root()
 	}
-
 	return root
 }

pkg/lang/ir/v1/r.go

@@ -21,17 +21,22 @@ import (
 	"github.com/moby/buildkit/client/llb"
 )
 
-func (g generalGraph) installRLang(root llb.State) llb.State {
-
-	installR := "apt-get update && apt-get install -y -t focal-cran40 r-base"
-
-	run := root.Run(llb.Shlexf("bash -c \"%s\"", installR),
+const rPath = "/usr/local/lib/R/site-library"
+
+func (g *generalGraph) installRLang(root llb.State) llb.State {
+	g.UserDirectories = append(g.UserDirectories, rPath)
+	prepare := root.Run(llb.Shlex(`sh -c "
+apt-get update && apt-get install -y --no-install-recommends --fix-missing gpg &&
+wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | gpg --dearmor -o /usr/share/keyrings/r-project.gpg &&
+echo "deb [signed-by=/usr/share/keyrings/r-project.gpg] https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/" | tee -a /etc/apt/sources.list.d/r-project.list
+"`), llb.WithCustomName("add R public GPG key")).Root()
+	run := prepare.Run(
+		llb.Shlex(`sh -c "apt-get update && apt-get install -y --no-install-recommends r-base"`),
 		llb.WithCustomNamef("[internal] apt install R environment from CRAN repository"))
 	return run.Root()
 }
 
 func (g generalGraph) installRPackages(root llb.State) llb.State {
-
 	if len(g.RPackages) == 0 {
 		return root
 	}
@@ -41,18 +46,11 @@ func (g generalGraph) installRPackages(root llb.State) llb.State {
 		mirrorURL = *g.CRANMirrorURL
 	}
 
-	lib := "/usr/local/lib/R/site-library/"
-
-	root = root.
-		Run(llb.Shlexf("chmod 777 %s", lib), llb.WithCustomNamef("[internal] setting execute permission for default R package library for envd users")).Root()
-
 	for _, packages := range g.RPackages {
-		command := fmt.Sprintf(`R -e 'options(repos = "%s"); install.packages(c("%s"), lib = "%s")'`, mirrorURL, strings.Join(packages, `","`), lib)
+		command := fmt.Sprintf(`R -e 'options(repos = "%s"); install.packages(c("%s"), lib = "%s")'`, mirrorURL, strings.Join(packages, `","`), rPath)
 		run := root.
 			Run(llb.Shlex(command), llb.WithCustomNamef("[internal] installing R packages: %s", strings.Join(packages, " ")))
 		root = run.Root()
-
 	}
-
 	return root
 }

pkg/lang/ir/v1/system.go

@@ -33,13 +33,6 @@ import (
 	"github.com/tensorchord/envd/pkg/util/fileutil"
 )
 
-// signFolder stores the path to the apt-source signature in string format
-// The value of signFolder should always be /etc/apt/keyrings
-const signFolder = "/etc/apt/keyrings"
-
-// signURI stores the third-party URI for downloading the signature of the corresponding repo
-const signURI = "https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc"
-
 func (g generalGraph) compileUbuntuAPT(root llb.State) llb.State {
 	if g.UbuntuAPTSource != nil {
 		logrus.WithField("source", *g.UbuntuAPTSource).Debug("using custom APT source")
@@ -53,89 +46,6 @@ func (g generalGraph) compileUbuntuAPT(root llb.State) llb.State {
 	return root
 }
 
-// copyAPTSignature returns the state and the path in string format
-// The returned string represents the location of the apt-source signature
-// A successful run of copyAPTSignature should return path of /etc/apt/keyrings/*.asc
-func (g generalGraph) copyAPTSignature(root llb.State, name string, url string) (llb.State, string) {
-
-	var fileName = fmt.Sprintf("%s.asc", name)
-
-	// path stores the location of the signature file
-	// The value of path should be /etc/apt/keyrings/*.asc
-	var path = filepath.Join(signFolder, fileName)
-
-	base := llb.Image(builderImage)
-	builder := base.
-		Run(llb.Shlexf("sh -c \"curl %s >> %s\"", url, fileName),
-			llb.WithCustomName("[internal] downloading apt-source signature in base image")).Root()
-
-	aptSign := root.
-		File(llb.Mkdir(signFolder, 0755, llb.WithParents(true)),
-			llb.WithCustomName("[internal] setting target apt-source signature folder")).
-		File(llb.Copy(builder, fileName, path),
-			llb.WithCustomName("[internal] copy signature from builder"))
-
-	return aptSign, path
-}
-
-// configRSrc returns the state and the content in DEB822 format for third-party apt-source
-// The returned string contains all configuration for adding third-party into apt source list
-// A successful run of configRSrc should return the content strictly follow the DEB822 format
-func (g generalGraph) configRSrc(root llb.State, aptConfig ir.APTConfig, sign string) (llb.State, string) {
-
-	var enabled = fmt.Sprintf("Enabled: %s\n", aptConfig.Enabled)
-	var types = fmt.Sprintf("Types: %s\n", aptConfig.Types)
-	var uris = fmt.Sprintf("URIs: %s\n", aptConfig.URIs)
-	var suites = fmt.Sprintf("Suites: %s\n", aptConfig.Suites)
-	var components = fmt.Sprintf("Components: %s\n", aptConfig.Components)
-	var architecture = fmt.Sprintf("Architectures: %s\n", aptConfig.Arch)
-
-	aptSign, signPath := g.copyAPTSignature(root, aptConfig.Name, sign)
-	var signature = fmt.Sprintf("Signed-By: %s\n", signPath)
-
-	var content strings.Builder
-	content.WriteString(enabled)
-	content.WriteString(types)
-	content.WriteString(uris)
-	content.WriteString(suites)
-	content.WriteString(components)
-	content.WriteString(signature)
-	content.WriteString(architecture)
-
-	return aptSign, content.String()
-}
-
-// compileRLang returns the llb.State only after compoiling the environment for installing R language
-// A successful run of compileRLang should set up the official R apt repo into /etc/apt/sources.list.d/*.sources
-func (g generalGraph) compileRLang(root llb.State) llb.State {
-
-	var aptConfig = ir.APTConfig{
-		Name:       "R-base",                                       // Name for the *.sources file in /etc/apt/sources.list.d
-		Enabled:    "yes",                                          // Represents the validation of the third-party repo
-		Types:      "deb",                                          // Type of the repo, binary or source code
-		URIs:       "https://cloud.r-project.org/bin/linux/ubuntu", // URI repo for the package
-		Suites:     "focal-cran40/",                                // Branch of the package
-		Components: "",                                             // Distribution of the package. E.g. main, non-free
-		Signed:     "R-base.asc",                                   // Name for the signature file
-		Arch:       "",                                             // Architecture that is supported
-	}
-
-	var file = fmt.Sprintf("/etc/apt/sources.list.d/%s.sources", aptConfig.Name)
-
-	aptSource, content := g.configRSrc(root, aptConfig, signURI)
-
-	aptRLang := aptSource.
-		File(llb.Mkdir("/etc/apt/sources.list.d/", 0755, llb.WithParents(true)),
-			llb.WithCustomName("[internal] setting apt-source folder sources.list.d")).
-		File(llb.Mkfile(file, 0644, []byte(content)),
-			llb.WithCustomName("[internal] setting apt-source file")).
-		File(llb.Mkfile("/etc/apt/apt.conf.d/DEB822.conf", 0644, []byte("APT::Sources::Use-Deb822 true;\n")),
-			llb.WithCustomName("[internal] setting apt-conf file to support DEB822 format"))
-
-	return aptRLang
-
-}
-
 func (g generalGraph) compileRun(root llb.State) llb.State {
 	if len(g.Exec) == 0 {
 		return root
@@ -259,8 +169,7 @@ func (g *generalGraph) compileLanguage(root llb.State) (llb.State, error) {
 			case "python":
 				root, err = g.installPython(root)
 			case "r":
-				rSrc := g.compileRLang(root)
-				root = g.installRLang(rSrc)
+				root = g.installRLang(root)
 			case "julia":
 				root = g.installJulia(root)
 			}
@@ -273,8 +182,7 @@ func (g *generalGraph) compileLanguage(root llb.State) (llb.State, error) {
 		case "python":
 			lang, err = g.installPython(root)
 		case "r":
-			rSrc := g.compileRLang(root)
-			lang = g.installRLang(rSrc)
+			lang = g.installRLang(root)
 		case "julia":
 			lang = g.installJulia(root)
 		}