From d581da42ec15cea35b705e4b499d03d01831551c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=90=AF=E8=88=AA?= <101104760+ZhangSetSail@users.noreply.github.com> Date: Mon, 4 Sep 2023 19:35:51 +0800 Subject: [PATCH] perf: optimize nodejsStatic build (#1747) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: optimize nodejsStatic build * fix: get package name failure * fix: modify some issues submitted this time --------- Co-authored-by: 张启航 --- builder/build/build.go | 3 +- builder/build/dockerfile_build.go | 12 +++++ builder/build/netcore_build.go | 51 +++++++++++++++++--- builder/exector/build_from_sourcecode_run.go | 9 ++++ builder/exector/groupapp_restore.go | 8 +-- builder/parser/code/lang.go | 3 ++ builder/parser/source_code.go | 29 ++++++++++- util/comman.go | 6 ++- 8 files changed, 106 insertions(+), 15 deletions(-) diff --git a/builder/build/build.go b/builder/build/build.go index df351eeb66..d6d167d8b4 100644 --- a/builder/build/build.go +++ b/builder/build/build.go @@ -40,7 +40,7 @@ func init() { buildcreaters = make(map[code.Lang]CreaterBuild) buildcreaters[code.Dockerfile] = dockerfileBuilder buildcreaters[code.Docker] = dockerfileBuilder - buildcreaters[code.NetCore] = netcoreBuilder + buildcreaters[code.NetCore] = customDockerBuilder buildcreaters[code.JavaJar] = slugBuilder buildcreaters[code.JavaMaven] = slugBuilder buildcreaters[code.JaveWar] = slugBuilder @@ -49,6 +49,7 @@ func init() { buildcreaters[code.Nodejs] = slugBuilder buildcreaters[code.Golang] = slugBuilder buildcreaters[code.OSS] = slugBuilder + buildcreaters[code.NodeJSDockerfile] = customDockerBuilder } var buildcreaters map[code.Lang]CreaterBuild diff --git a/builder/build/dockerfile_build.go b/builder/build/dockerfile_build.go index ec1e8c2784..a187a76d63 100644 --- a/builder/build/dockerfile_build.go +++ b/builder/build/dockerfile_build.go @@ -211,6 +211,14 @@ func (d *dockerfileBuild) createVolumeAndMount(re *Request, secretName, ServiceI } volumes = []corev1.Volume{ dockerfileBuildVolume, + { + Name: "grdata", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "rbd-cpt-grdata", + }, + }, + }, { Name: "buildkittoml", VolumeSource: corev1.VolumeSource{ @@ -250,6 +258,10 @@ func (d *dockerfileBuild) createVolumeAndMount(re *Request, secretName, ServiceI }, } volumeMounts = []corev1.VolumeMount{ + { + Name: "grdata", + MountPath: "/grdata", + }, { Name: "dockerfile-build", MountPath: "/cache", diff --git a/builder/build/netcore_build.go b/builder/build/netcore_build.go index 01275f6335..a9784ca0a7 100644 --- a/builder/build/netcore_build.go +++ b/builder/build/netcore_build.go @@ -20,6 +20,7 @@ package build import ( "fmt" + "github.com/goodrain/rainbond/builder/parser/code" "io/ioutil" "os" "path" @@ -44,7 +45,29 @@ COPY --from=builder /out/ . CMD ["dotnet"] ` -type netcoreBuild struct { +var nodeJSStaticDockerfileTmpl = ` +FROM node:${RUNTIMES:20}-bullseye-slim AS builder +COPY . /app +WORKDIR /app +RUN ${PACKAGE_TOOL:npm} config set registry ${NPM_REGISTRY:https://registry.npmmirror.com} && ${PACKAGE_TOOL:npm} install && ${NODE_BUILD_CMD:npm run build} + +FROM nginx:alpine +COPY nginx.k8s.conf /etc/nginx/conf.d/default.conf + +COPY --from=builder /app/${DIST_DIR:dist} /opt/${DIST_DIR:dist} +` + +var nginxTemplate = ` +server { + listen 5000; + + location / { + root /opt/${DIST_DIR:dist}; + } +} +` + +type customDockerfileBuild struct { imageName string buildImageName string sourceDir string @@ -53,11 +76,11 @@ type netcoreBuild struct { imageClient sources.ImageClient } -func netcoreBuilder() (Build, error) { - return &netcoreBuild{}, nil +func customDockerBuilder() (Build, error) { + return &customDockerfileBuild{}, nil } -func (d *netcoreBuild) Build(re *Request) (*Response, error) { +func (d *customDockerfileBuild) Build(re *Request) (*Response, error) { defer d.clear() d.logger = re.Logger d.serviceID = re.ServiceID @@ -67,7 +90,7 @@ func (d *netcoreBuild) Build(re *Request) (*Response, error) { re.Logger.Info("start compiling the source code", map[string]string{"step": "builder-exector"}) // write dockerfile - if err := d.writeDockerfile(d.sourceDir, re.BuildEnvs); err != nil { + if err := d.writeDockerfile(d.sourceDir, re.BuildEnvs, re.Lang); err != nil { return nil, fmt.Errorf("write default dockerfile error:%s", err.Error()) } // build image @@ -84,20 +107,32 @@ func (d *netcoreBuild) Build(re *Request) (*Response, error) { return d.createResponse(), nil } -func (d *netcoreBuild) writeDockerfile(sourceDir string, envs map[string]string) error { +func (d *customDockerfileBuild) writeDockerfile(sourceDir string, envs map[string]string, lang code.Lang) error { dockerfile := util.ParseVariable(dockerfileTmpl, envs) + if lang == "NodeJSStatic" && envs["MODE"] == "DOCKERFILE" { + if envs["NODE_BUILD_CMD"] == "" { + envs["NODE_BUILD_CMD"] = envs["PACKAGE_TOOL"] + " run build" + } + dockerfile = util.ParseVariable(nodeJSStaticDockerfileTmpl, envs) + dPath := path.Join(sourceDir, "nginx.k8s.conf") + nginxFile := util.ParseVariable(nginxTemplate, envs) + err := ioutil.WriteFile(dPath, []byte(nginxFile), 0755) + if err != nil { + return err + } + } dfpath := path.Join(sourceDir, "Dockerfile") logrus.Debugf("dest: %s; write dockerfile: %s", dfpath, dockerfile) return ioutil.WriteFile(dfpath, []byte(dockerfile), 0755) } -func (d *netcoreBuild) createResponse() *Response { +func (d *customDockerfileBuild) createResponse() *Response { return &Response{ MediumType: ImageMediumType, MediumPath: d.imageName, } } -func (d *netcoreBuild) clear() { +func (d *customDockerfileBuild) clear() { os.Remove(path.Join(d.sourceDir, "Dockerfile")) } diff --git a/builder/exector/build_from_sourcecode_run.go b/builder/exector/build_from_sourcecode_run.go index a2faf2d99e..166ff6d2c0 100644 --- a/builder/exector/build_from_sourcecode_run.go +++ b/builder/exector/build_from_sourcecode_run.go @@ -229,6 +229,12 @@ func (i *SourceCodeBuildItem) Run(timeout time.Duration) error { } if len(packageArr) != 0 { fileName := packageArr[0] + for _, pa := range packageArr { + paExt := path.Ext(pa) + if strings.HasSuffix(paExt, `.zip`) || strings.HasSuffix(paExt, `.tar`) || strings.HasSuffix(paExt, `.tar.gz`) || strings.HasSuffix(paExt, `.jar`) || strings.HasSuffix(paExt, `.war`) { + fileName = pa + } + } file := filePath + "/" + fileName fileMD5 := util.MD5(file) i.commit = Commit{ @@ -317,6 +323,9 @@ func (i *SourceCodeBuildItem) Run(timeout time.Duration) error { func (i *SourceCodeBuildItem) codeBuild() (*build.Response, error) { codeBuild, err := build.GetBuild(code.Lang(i.Lang)) + if i.Lang == "NodeJSStatic" && i.BuildEnvs["MODE"] == "DOCKERFILE" { + codeBuild, err = build.GetBuild(code.NodeJSDockerfile) + } if err != nil { logrus.Errorf("get code build error: %s lang %s", err.Error(), i.Lang) i.Logger.Error(util.Translation("No way of compiling to support this source type was found"), map[string]string{"step": "builder-exector", "status": "failure"}) diff --git a/builder/exector/groupapp_restore.go b/builder/exector/groupapp_restore.go index 6c783f51e5..6a56677007 100644 --- a/builder/exector/groupapp_restore.go +++ b/builder/exector/groupapp_restore.go @@ -228,7 +228,7 @@ func (b *BackupAPPRestore) restoreVersionAndData(backup *dbmodel.AppBackup, appS allTmpDir := fmt.Sprintf("/grdata/tmp/%s", app.ServiceID) if exist, _ := util.FileExists(allDataFilePath); exist { logrus.Infof("unzip all data from %s to %s", allDataFilePath, allTmpDir) - if err := util.Unzip(allDataFilePath, allTmpDir); err != nil { + if err := util.Unzip(allDataFilePath, allTmpDir, false); err != nil { logrus.Errorf("unzip all data file failure %s", err.Error()) } else { allDataRestore = true @@ -243,7 +243,7 @@ func (b *BackupAPPRestore) restoreVersionAndData(backup *dbmodel.AppBackup, appS dstDir := fmt.Sprintf("%s/data_%s/%s.zip", b.cacheDir, b.getOldServiceID(app.ServiceID), strings.Replace(volume.VolumeName, "/", "", -1)) tmpDir = fmt.Sprintf("/grdata/tmp/%s_%d", volume.ServiceID, volume.ID) logrus.Infof("unzip %s to %s", dstDir, tmpDir) - if err := util.Unzip(dstDir, tmpDir); err != nil { + if err := util.Unzip(dstDir, tmpDir, false); err != nil { if !strings.Contains(err.Error(), "no such file") { logrus.Errorf("restore service(%s) volume(%s) data error.%s", app.ServiceID, volume.VolumeName, err.Error()) return err @@ -712,7 +712,7 @@ func (b *BackupAPPRestore) restoreMetadata(appSnapshot *AppSnapshot) error { func (b *BackupAPPRestore) downloadFromLocal(backup *dbmodel.AppBackup) error { sourceDir := backup.SourceDir - err := util.Unzip(sourceDir, b.cacheDir) + err := util.Unzip(sourceDir, b.cacheDir, false) if err != nil { b.Logger.Error(util.Translation("unzip metadata file error"), map[string]string{"step": "backup_builder", "status": "failure"}) logrus.Errorf("unzip file error when restore backup app , %s", err.Error()) @@ -752,7 +752,7 @@ func (b *BackupAPPRestore) downloadFromS3(sourceDir string) error { } logrus.Debugf("successfully downloading backup file: %s", disDir) - err = util.Unzip(disDir, b.cacheDir) + err = util.Unzip(disDir, b.cacheDir, false) if err != nil { // b.Logger.Error(util.Translation("unzip metadata file error"), map[string]string{"step": "backup_builder", "status": "failure"}) logrus.Errorf("error unzipping backup file: %v", err) diff --git a/builder/parser/code/lang.go b/builder/parser/code/lang.go index 90005f69d2..9f1786c50d 100644 --- a/builder/parser/code/lang.go +++ b/builder/parser/code/lang.go @@ -94,6 +94,9 @@ var JavaJar Lang = "Java-jar" //Nodejs Lang var Nodejs Lang = "Node.js" +//NodeJSDockerfile Lang +var NodeJSDockerfile Lang = "NodeJSDockerfile" + //NodeJSStatic static Lang var NodeJSStatic Lang = "NodeJSStatic" diff --git a/builder/parser/source_code.go b/builder/parser/source_code.go index f2b1c11749..7f0edb87e3 100644 --- a/builder/parser/source_code.go +++ b/builder/parser/source_code.go @@ -220,6 +220,30 @@ func (d *SourceCodeParse) Parse() ParseErrorList { checkPath = fmt.Sprintf("/grdata/package_build/temp/events/%s", eventID) } buildInfo.CodeHome = checkPath + + fileList, err := ioutil.ReadDir(buildInfo.GetCodeHome()) + var ext, filePath string + if len(fileList) > 0 { + filePath = path.Join(buildInfo.GetCodeHome(), fileList[0].Name()) + ext = path.Ext(fileList[0].Name()) + } + switch ext { + case ".tar": + if err := util.UnTar(filePath, buildInfo.GetCodeHome(), false); err != nil { + logrus.Errorf("untar package file failure %s", err.Error()) + d.errappend(ErrorAndSolve(FatalError, "文件解压失败", "请确认该文件是否为tar规范文件")) + } + case ".tgz", ".tar.gz": + if err := util.UnTar(filePath, buildInfo.GetCodeHome(), true); err != nil { + logrus.Errorf("untar package file failure %s", err.Error()) + d.errappend(ErrorAndSolve(FatalError, "文件解压失败", "请确认该文件是否为tgz规范文件")) + } + case ".zip": + if err := util.Unzip(filePath, buildInfo.GetCodeHome(), true); err != nil { + logrus.Errorf("untar package file failure %s", err.Error()) + d.errappend(ErrorAndSolve(FatalError, "文件解压失败", "请确认该文件是否为zip规范文件")) + } + } return ParseErrorList{} } ossFunc := func() ParseErrorList { @@ -256,7 +280,7 @@ func (d *SourceCodeParse) Parse() ParseErrorList { d.errappend(ErrorAndSolve(FatalError, "文件解压失败", "请确认该文件是否为tgz规范文件")) } case ".zip": - if err := util.Unzip(fileName, buildInfo.GetCodeHome()); err != nil { + if err := util.Unzip(fileName, buildInfo.GetCodeHome(), false); err != nil { logrus.Errorf("untar package file failure %s", err.Error()) d.errappend(ErrorAndSolve(FatalError, "文件解压失败", "请确认该文件是否为zip规范文件")) } @@ -537,6 +561,9 @@ func (d *SourceCodeParse) GetImage() Image { //GetArgs 启动参数 func (d *SourceCodeParse) GetArgs() []string { + if d.Lang == code.Nodejs { + return nil + } return d.args } diff --git a/util/comman.go b/util/comman.go index dc560413d9..f17eabed34 100644 --- a/util/comman.go +++ b/util/comman.go @@ -539,7 +539,7 @@ func UnTar(archive, target string, zip bool) error { } //Unzip archive file to target dir -func Unzip(archive, target string) error { +func Unzip(archive, target string, currentDirectory bool) error { reader, err := zip.OpenDirectReader(archive) if err != nil { return fmt.Errorf("error opening archive: %v", err) @@ -550,6 +550,10 @@ func Unzip(archive, target string) error { for _, file := range reader.File { run := func() error { path := filepath.Join(target, file.Name) + if currentDirectory { + p := strings.Split(file.Name, "/")[1:] + path = filepath.Join(target, strings.Join(p, "/")) + } if file.FileInfo().IsDir() { os.MkdirAll(path, file.Mode()) if file.Comment != "" && strings.Contains(file.Comment, "/") {