Skip to content

Commit

Permalink
Revert "POC: Speed up compilation by freezing container during compil…
Browse files Browse the repository at this point in the history
…ation. (google#11940)"

This reverts commit deef8c5.
  • Loading branch information
manunio committed Jun 24, 2024
1 parent c433f95 commit 25d2195
Show file tree
Hide file tree
Showing 5 changed files with 67 additions and 416 deletions.
6 changes: 2 additions & 4 deletions infra/base-images/base-builder/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -170,10 +170,8 @@ COPY bazel_build_fuzz_tests \

# TODO: Build this as part of a multi-stage build.
ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang-jcc /usr/local/bin/
ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang++-jcc /usr/local/bin
ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang-jcc2 /usr/local/bin/
ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang++-jcc2 /usr/local/bin
RUN chmod +x /usr/local/bin/clang-jcc /usr/local/bin/clang++-jcc /usr/local/bin/clang-jcc2 /usr/local/bin/clang++-jcc2
ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang++-jcc /usr/local/bin/
RUN chmod +x /usr/local/bin/clang-jcc && chmod +x /usr/local/bin/clang++-jcc

COPY llvmsymbol.diff $SRC
COPY detect_repo.py /opt/cifuzz/
Expand Down
6 changes: 2 additions & 4 deletions infra/base-images/base-builder/jcc/build_jcc.bash
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
################################################################################

go build jcc.go
go build jcc2.go
cp jcc clang
cp jcc clang++
gsutil cp jcc gs://clusterfuzz-builds/jcc/clang++-jcc
gsutil cp jcc gs://clusterfuzz-builds/jcc/clang-jcc

gsutil cp jcc2 gs://clusterfuzz-builds/jcc/clang++-jcc2
gsutil cp jcc2 gs://clusterfuzz-builds/jcc/clang-jcc2
64 changes: 63 additions & 1 deletion infra/base-images/base-builder/jcc/jcc.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"os/exec"
"path/filepath"
"regexp"
"slices"
"strings"
)

Expand Down Expand Up @@ -168,6 +169,63 @@ func CorrectMissingHeaders(bin string, cmd []string) ([]string, bool, error) {
return cmd, false, nil
}

func EnsureDir(dirPath string) {
// Checks if a path is an existing directory, otherwise create one.
if pathInfo, err := os.Stat(dirPath); err == nil {
if isDir := pathInfo.IsDir(); !isDir {
panic(dirPath + " exists but is not a directory.")
}
} else if errors.Is(err, fs.ErrNotExist) {
if err := os.MkdirAll(dirPath, 0755); err != nil {
panic("Failed to create directory: " + dirPath + ".")
}
fmt.Println("Created directory: " + dirPath + ".")
} else {
panic("An error occurred in os.Stat(" + dirPath + "): " + err.Error())
}
}

func GenerateAST(bin string, args []string, filePath string) {
// Generates AST.
outFile, err := os.Create(filePath)
if err != nil {
fmt.Println(err)
}
defer outFile.Close()

cmd := exec.Command(bin, args...)
cmd.Stdout = outFile
cmd.Run()
}

func GenerateASTs(bin string, args []string, astDir string) {
// Generates an AST for each C/CPP file in the command.
// Cannot save AST when astDir is not available.
EnsureDir(astDir)

// Target file suffixes.
suffixes := []string{".cpp", ".cc", ".cxx", ".c++", ".c", ".h", ".hpp"}
// C/CPP targets in the command.
targetFiles := []string{}
// Flags to generate AST.
flags := []string{"-Xclang", "-ast-dump=json", "-fsyntax-only"}
for _, arg := range args {
targetFileExt := strings.ToLower(filepath.Ext(arg))
if slices.Contains(suffixes, targetFileExt) {
targetFiles = append(targetFiles, arg)
continue
}
flags = append(flags, arg)
}

// Generate an AST for each target file. Skips AST generation when a
// command has no target file (e.g., during linking).
for _, targetFile := range targetFiles {
filePath := filepath.Join(astDir, fmt.Sprintf("%s.ast", filepath.Base(targetFile)))
GenerateAST(bin, append(flags, targetFile), filePath)
}
}

func ExecBuildCommand(bin string, args []string) (int, string, string) {
// Executes the original command.
cmd := exec.Command(bin, args...)
Expand All @@ -180,6 +238,10 @@ func ExecBuildCommand(bin string, args []string) (int, string, string) {
}

func Compile(bin string, args []string) (int, string, string) {
// Generate ASTs f we define this ENV var.
if astDir := os.Getenv("JCC_GENERATE_AST_DIR"); astDir != "" {
GenerateASTs(bin, args, astDir)
}
// Run the actual command.
return ExecBuildCommand(bin, args)
}
Expand Down Expand Up @@ -298,7 +360,7 @@ func WriteStdErrOut(args []string, outstr string, errstr string) {
fmt.Print(outstr)
fmt.Fprint(os.Stderr, errstr)
// Record what compile args produced the error and the error itself in log file.
AppendStringToFile("/workspace/err.log", fmt.Sprintf("%s\n", args)+errstr)
AppendStringToFile("/workspace/err.log", fmt.Sprintf("%s\n", args) + errstr)
}

func main() {
Expand Down
Loading

0 comments on commit 25d2195

Please sign in to comment.