diff --git a/cmd/cloudexec/push.go b/cmd/cloudexec/push.go index 22a8c14..8a867ed 100644 --- a/cmd/cloudexec/push.go +++ b/cmd/cloudexec/push.go @@ -1,107 +1,107 @@ package main import ( - "io" + "archive/zip" "fmt" + "io" "os" "path/filepath" - "archive/zip" "github.com/crytic/cloudexec/pkg/config" "github.com/crytic/cloudexec/pkg/s3" ) func UploadDirectoryToSpaces(config config.Config, bucket string, sourcePath string, destPath string) error { - // Compute the path for the zipped archive of sourcePath - zipFileName := "input.zip" - zipFilePath, err := filepath.Abs(filepath.Join(filepath.Dir(sourcePath), zipFileName)) - if err != nil { - return err - } + // Compute the path for the zipped archive of sourcePath + zipFileName := "input.zip" + zipFilePath, err := filepath.Abs(filepath.Join(filepath.Dir(sourcePath), zipFileName)) + if err != nil { + return err + } - // Create a file where we will write the zipped archive - zipFile, err := os.Create(zipFilePath) - if err != nil { - return err - } - defer zipFile.Close() + // Create a file where we will write the zipped archive + zipFile, err := os.Create(zipFilePath) + if err != nil { + return err + } + defer zipFile.Close() - // Create a new zip writer - zipWriter := zip.NewWriter(zipFile) - defer zipWriter.Close() + // Create a new zip writer + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() // Walk the directory and recursively add files to the zipped archive - err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - // If this is a subdirectory, make sure the path ends with a trailing slash before we create it - // See https://pkg.go.dev/archive/zip#Writer.Create for details + // If this is a subdirectory, make sure the path ends with a trailing slash before we create it + // See https://pkg.go.dev/archive/zip#Writer.Create for details if info.IsDir() { - cleanPath := filepath.Clean(path) + string(filepath.Separator) - _, err := zipWriter.Create(cleanPath) - if err != nil { - return err - } - fmt.Printf("Created directory %s in the zipped archive\n", cleanPath) - return nil - } + cleanPath := filepath.Clean(path) + string(filepath.Separator) + _, err := zipWriter.Create(cleanPath) + if err != nil { + return err + } + fmt.Printf("Created directory %s in the zipped archive\n", cleanPath) + return nil + } - // Create a new file entry in the zipped archive - zipFileEntry, err := zipWriter.Create(path) - if err != nil { - return err - } + // Create a new file entry in the zipped archive + zipFileEntry, err := zipWriter.Create(path) + if err != nil { + return err + } - // Open the file we're adding to the zipped archive - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() + // Open the file we're adding to the zipped archive + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() - // Write this file to the zipped archive - _, err = io.Copy(zipFileEntry, file) - if err != nil { - return err - } + // Write this file to the zipped archive + _, err = io.Copy(zipFileEntry, file) + if err != nil { + return err + } - fmt.Printf("Added %s to the zipped archive\n", path) + fmt.Printf("Added %s to the zipped archive\n", path) return nil }) - if err != nil { - return err - } - fmt.Printf("Successfully added all files from %s to zipped archive at %s\n", sourcePath, zipFilePath) + if err != nil { + return err + } + fmt.Printf("Successfully added all files from %s to zipped archive at %s\n", sourcePath, zipFilePath) - // Make sure all prior writes are sync'd to the filesystem - // This is necessary because we're going to read the file immediately after writing it - err = zipWriter.Flush() - if err != nil { - return err - } - err = zipFile.Sync() - if err != nil { - return err - } + // Make sure all prior writes are sync'd to the filesystem + // This is necessary because we're going to read the file immediately after writing it + err = zipWriter.Flush() + if err != nil { + return err + } + err = zipFile.Sync() + if err != nil { + return err + } - // Read the zipped archive - fileBytes, err := os.ReadFile(zipFilePath) - if err != nil { - return fmt.Errorf("Failed to read zipped archive %s: %w", zipFilePath, err) - } - if len(fileBytes) == 0 { - return fmt.Errorf("Failed to read zipped archive at %s: read zero bytes of data", zipFilePath) - } + // Read the zipped archive + fileBytes, err := os.ReadFile(zipFilePath) + if err != nil { + return fmt.Errorf("Failed to read zipped archive %s: %w", zipFilePath, err) + } + if len(fileBytes) == 0 { + return fmt.Errorf("Failed to read zipped archive at %s: read zero bytes of data", zipFilePath) + } - // Upload the zipped archive - destKey := filepath.Join(destPath, "input.zip") - fmt.Printf("Uploading archive (%v bytes) to %s\n", len(fileBytes), destKey) - err = s3.PutObject(config, bucket, destKey, fileBytes) - if err != nil { - return err - } + // Upload the zipped archive + destKey := filepath.Join(destPath, "input.zip") + fmt.Printf("Uploading archive (%v bytes) to %s\n", len(fileBytes), destKey) + err = s3.PutObject(config, bucket, destKey, fileBytes) + if err != nil { + return err + } - return nil + return nil } diff --git a/cmd/cloudexec/user_data.sh.tmpl b/cmd/cloudexec/user_data.sh.tmpl index 7ff0530..56fc39d 100644 --- a/cmd/cloudexec/user_data.sh.tmpl +++ b/cmd/cloudexec/user_data.sh.tmpl @@ -126,12 +126,13 @@ cleanup() { update_state "failed" fi - if [ -n "$(ls -A ~/output/)" ]; then - echo "Uploading results..." - s3cmd put -r ~/output/* "s3://${BUCKET_NAME}/job-${JOB_ID}/output/" || true # proceed to destroy droplet on error - else - echo "Skipping results upload, no files found in ~/ouput" - fi + output_files="$(ls -A ~/output/)" + if [[ -n ${output_files} ]]; then + echo "Uploading results..." + s3cmd put -r ~/output/* "s3://${BUCKET_NAME}/job-${JOB_ID}/output/" + else + echo "Skipping results upload, no files found in ~/ouput" + fi if [[ -s ${stdout_log} ]]; then echo @@ -150,11 +151,11 @@ cleanup() { fi if [[ -s "/var/log/cloud-init-output.log" ]]; then - echo "Uploading logs..." - s3cmd put /var/log/cloud-init-output.log "s3://${BUCKET_NAME}/job-${JOB_ID}/logs/" - else - echo "No logs to upload.." - fi + echo "Uploading logs..." + s3cmd put /var/log/cloud-init-output.log "s3://${BUCKET_NAME}/job-${JOB_ID}/logs/" + else + echo "No logs to upload.." + fi echo echo "Destroying droplet..." @@ -209,9 +210,9 @@ if [[ ! -s ~/input.zip ]]; then fi echo "Unzipping input archive..." -ls -l ~/ -ls -l ~/input.zip -xxd ~/input.zip | head -n 10 +#ls -l ~/ +#ls -l ~/input.zip +#xxd ~/input.zip | head -n 10 unzip ~/input.zip -d ~/ ls ~/ ls ~/input diff --git a/pkg/s3/s3.go b/pkg/s3/s3.go index 3b30ed3..d1f3620 100644 --- a/pkg/s3/s3.go +++ b/pkg/s3/s3.go @@ -159,7 +159,7 @@ func PutObject(config config.Config, bucket string, key string, value []byte) er if err != nil { return fmt.Errorf("Failed to create %s directory in bucket %s: %w", key, bucket, err) } - fmt.Printf("Successfully created directory in s3 bucket: %s/%s\n", bucket, key) + fmt.Printf("Successfully created directory in s3 bucket: %s/%s\n", bucket, key) return nil } @@ -181,7 +181,7 @@ func PutObject(config config.Config, bucket string, key string, value []byte) er return fmt.Errorf("Failed to upload file %s to bucket %s: %w", key, bucket, err) } - fmt.Printf("Successfully uploaded file to s3 bucket: %s/%s\n", bucket, key) + fmt.Printf("Successfully uploaded file to s3 bucket: %s/%s\n", bucket, key) return nil }