From 7f0733e614fe62bd54fcc03311ba5011e56e01cc Mon Sep 17 00:00:00 2001 From: bohendo Date: Tue, 19 Sep 2023 15:00:22 -0400 Subject: [PATCH 01/22] consolidate bucketName creation --- cmd/cloudexec/init.go | 7 +++---- cmd/cloudexec/launch.go | 2 +- cmd/cloudexec/main.go | 3 ++- pkg/s3/s3.go | 5 +---- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cmd/cloudexec/init.go b/cmd/cloudexec/init.go index 990c188..f60049a 100644 --- a/cmd/cloudexec/init.go +++ b/cmd/cloudexec/init.go @@ -7,12 +7,11 @@ import ( "github.com/trailofbits/cloudexec/pkg/s3" ) -func Init(username string, config config.Config) error { - bucketName := fmt.Sprintf("cloudexec-%s-trailofbits", username) +func Init(bucketName string, config config.Config) error { // Create a new bucket (or get an existing one) - err := s3.GetOrCreateBucket(config, username) + err := s3.GetOrCreateBucket(config, bucketName) if err != nil { - return fmt.Errorf("Failed to get bucket for %s: %w", username, err) + return fmt.Errorf("Failed to get %s bucket: %w", bucketName, err) } fmt.Printf("Using bucket: %v\n", bucketName) diff --git a/cmd/cloudexec/launch.go b/cmd/cloudexec/launch.go index 7bb6c33..0bcba1c 100644 --- a/cmd/cloudexec/launch.go +++ b/cmd/cloudexec/launch.go @@ -90,7 +90,7 @@ func Launch(user *user.User, config config.Config, dropletSize string, dropletRe bucketName := fmt.Sprintf("cloudexec-%s-trailofbits", username) fmt.Printf("Getting or creating new bucket for %s...\n", username) - err := s3.GetOrCreateBucket(config, username) + err := s3.GetOrCreateBucket(config, bucketName) if err != nil { return fmt.Errorf("Failed to get bucket for %s: %w", username, err) } diff --git a/cmd/cloudexec/main.go b/cmd/cloudexec/main.go index cb5c187..cb0f34c 100644 --- a/cmd/cloudexec/main.go +++ b/cmd/cloudexec/main.go @@ -31,6 +31,7 @@ func main() { os.Exit(1) } userName := user.Username + // TODO: sanitize username usage in bucketname bucketName := fmt.Sprintf("cloudexec-%s-trailofbits", userName) // Attempt to load the configuration @@ -73,7 +74,7 @@ func main() { return configErr } - err = Init(userName, config) + err = Init(bucketName, config) if err != nil { return err } diff --git a/pkg/s3/s3.go b/pkg/s3/s3.go index 4c47cf3..894cc98 100644 --- a/pkg/s3/s3.go +++ b/pkg/s3/s3.go @@ -95,10 +95,7 @@ func ListBuckets(config config.Config) ([]string, error) { return buckets, nil } -func GetOrCreateBucket(config config.Config, username string) error { - // TODO: sanitize username & centralize bucket name creation - bucket := fmt.Sprintf("cloudexec-%s-trailofbits", username) - +func GetOrCreateBucket(config config.Config, bucket string) error { listBucketsOutput, err := ListBuckets(config) if err != nil { return fmt.Errorf("Failed to list buckets: %w", err) From a1a9fda6090b061b258d4a2f6dc5aa0b54558fc7 Mon Sep 17 00:00:00 2001 From: bohendo Date: Tue, 19 Sep 2023 15:54:12 -0400 Subject: [PATCH 02/22] auto-init before state access & rm init command --- README.md | 84 ++++++++++++++---------------- cmd/cloudexec/clean.go | 6 +-- cmd/cloudexec/configure.go | 2 +- cmd/cloudexec/init.go | 4 +- cmd/cloudexec/launch.go | 15 ++---- cmd/cloudexec/logs.go | 4 +- cmd/cloudexec/main.go | 90 ++++++++++++++++++++++++--------- cmd/cloudexec/pull.go | 4 +- cmd/cloudexec/push.go | 4 +- cmd/cloudexec/user_data.go | 2 +- cmd/cloudexec/user_data_test.go | 2 +- 11 files changed, 124 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index cc4d1ce..561b874 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# CloudFuzz +# CloudExec -CloudFuzz is a command-line tool for easily running cloud-based fuzzing jobs on DigitalOcean. Start, manage and pull the results of fuzzing jobs from your terminal. +CloudExec is a command-line tool for easily running cloud-based jobs on DigitalOcean. Start, manage and pull the results of jobs from your terminal. ## Getting Started @@ -10,20 +10,20 @@ CloudFuzz is a command-line tool for easily running cloud-based fuzzing jobs on ```bash brew tap trailofbits/tools -brew install cloudfuzz +brew install cloudexec ``` #### Upgrade with Brew ```bash -brew update && brew upgrade cloudfuzz +brew update && brew upgrade cloudexec ``` alternatively, you can install from a GitHub release: ### Install from a GitHub release -Download the latest release for your platform from the [releases page](https://github.com/trailofbits/cloudfuzz/releases). +Download the latest release for your platform from the [releases page](https://github.com/crytic/cloudexec/releases). #### Release verification @@ -31,36 +31,36 @@ Releases are signed with sigstore. You can verify using [`cosign`](https://githu ```bash cosign verify-blob \ - --certificate-identity-regexp "https://github.com/trailofbits/cloudfuzz.*" \ + --certificate-identity-regexp "https://github.com/crytic/cloudexec.*" \ --certificate-oidc-issuer https://token.actions.githubusercontent.com \ - --bundle cloudfuzz---.tar.gz.bundle \ - cloudfuzz---.tar.gz + --bundle cloudexec---.tar.gz.bundle \ + cloudexec---.tar.gz ``` #### Install from a tarball ```bash -tar -xzf cloudfuzz---.tar.gz -mv cloudfuzz /usr/local/bin +tar -xzf cloudexec---.tar.gz +mv cloudexec /usr/local/bin ``` #### Install from source -Running the command below will build the CLI tool from source with a binary named `cloudfuzz` in a `dist` folder: +Running the command below will build the CLI tool from source with a binary named `cloudexec` in a `dist` folder: ```bash make build ``` -Then, move the resulting binary from `./dist/cloufuzz` into your `PATH`. +Then, move the resulting binary from `./dist/clouexec` into your `PATH`. -Nix users can run `nix build` and then `nix profile install ./result` to install `cloudfuzz`. A helper command `make nix-install` is available which performs these steps for you and also upgrades an existing version of `cloudfuzz` that might already be installed. +Nix users can run `nix build` and then `nix profile install ./result` to install `cloudexec`. A helper command `make nix-install` is available which performs these steps for you and also upgrades an existing version of `cloudexec` that might already be installed. ### Configure credentials -CloudFuzz requires DigitalOcean API credentials to manage droplets, and Spaces credentials to store state and job data. The recommended method for storing and providing your credentials securely is by using the 1Password CLI. +CloudExec requires DigitalOcean API credentials to manage droplets, and Spaces credentials to store state and job data. The recommended method for storing and providing your credentials securely is by using the 1Password CLI. -CloudFuzz supports natively integrating with 1Password, allowing you to reference your credentials stored in your 1Password vault. However, you can also choose to provide plaintext credentials using the `cloudfuzz configure` command. Additionally, you can override individual values or the entire configuration by setting the corresponding environment variables. +CloudExec supports natively integrating with 1Password, allowing you to reference your credentials stored in your 1Password vault. However, you can also choose to provide plaintext credentials using the `cloudexec configure` command. Additionally, you can override individual values or the entire configuration by setting the corresponding environment variables. #### Get credentials from DigitalOcean @@ -82,14 +82,14 @@ brew install --cask 1password/tap/1password-cli # see the link above for install eval $(op signin) ``` -Note what your [1Password secret references](https://developer.1password.com/docs/cli/secret-references/) are and use them in place of your actual secret values during the `cloudfuzz configure` or env var setup steps described in the next section. +Note what your [1Password secret references](https://developer.1password.com/docs/cli/secret-references/) are and use them in place of your actual secret values during the `cloudexec configure` or env var setup steps described in the next section. These references generally follow the format: `op:////`. For example, if you saved your keys to a vault called `Private`, in an item called `DigitalOcean` and the api key field is called `ApiKey`, then the secret reference to use is `op://Private/DigitalOcean/ApiKey`. -#### Configure CloudFuzz +#### Configure CloudExec ```bash -cloudfuzz configure +cloudexec configure ``` or set environment variables: @@ -103,41 +103,35 @@ DIGITALOCEAN_SPACES_REGION Remember, if you save secret values to a `.env` file, never commit it to any version control system. Add such `.env` files to your project's `.gitignore` file to help prevent making such mistakes. -### Check CloudFuzz access +### Check CloudExec access -Confirm `cloudfuzz` has access to DigitalOcean. +Confirm `cloudexec` has access to DigitalOcean. ```bash -cloudfuzz check +cloudexec check ``` -### Initialize your CloudFuzz environment +### Launch a new remote job -```bash -cloudfuzz init -``` - -### Launch a new remote fuzzing job - -Generate a cloudfuzz.toml configuration file in the current directory. +Generate a cloudexec.toml configuration file in the current directory. ```bash -cloudfuzz launch init +cloudexec launch init ``` -Update the `cloudfuzz.toml` as needed. +Update the `cloudexec.toml` as needed. ```bash -# default nyc3 region and c-2 size droplet, using a cloudfuzz.toml file in the current directory -cloudfuzz launch +# default nyc3 region and c-2 size droplet, using a cloudexec.toml file in the current directory +cloudexec launch # custom region and droplet size -cloudfuzz launch --size c-4 --region sfo2 +cloudexec launch --size c-4 --region sfo2 ``` ### Stream logs from the provisioning script ```bash -cloudfuzz logs +cloudexec logs ``` Note that the `logs` subcommand will continue to stream logs until you stop with ctrl-c, even after the job is finished and stops producing new logs. This is a read-only command and it is safe to kill it at any point. @@ -145,53 +139,53 @@ Note that the `logs` subcommand will continue to stream logs until you stop with ### Get logs from a previous run ```bash -cloudfuzz logs --job 1 +cloudexec logs --job 1 ``` ### Attach to the running job ```bash -cloudfuzz attach +cloudexec attach # or -ssh -t cloudfuzz tmux attach -s cloudfuzz +ssh -t cloudexec tmux attach -s cloudexec ``` ### SSH to your droplet ```bash -ssh cloudfuzz +ssh cloudexec ``` ### Check on the status of your jobs ```bash # show only runnning jobs, and the last completed job -cloudfuzz status +cloudexec status # show all jobs -cloudfuzz status --all +cloudexec status --all ``` ### Sync files from a completed job to a local path ```bash # pull from the latest successful job -cloudfuzz pull example/output +cloudexec pull example/output # pull from any job ID -cloudfuzz pull --job 1 example/output +cloudexec pull --job 1 example/output ``` ### Cancel any in progress jobs ```bash -cloudfuzz cancel +cloudexec cancel ``` ### Cleanup all bucket contents and reset state (destructive) ```bash -cloudfuzz clean +cloudexec clean ``` Note that there is often a delay while deleting files from Digital Ocean Spaces buckets. diff --git a/cmd/cloudexec/clean.go b/cmd/cloudexec/clean.go index 4586598..05a34d6 100644 --- a/cmd/cloudexec/clean.go +++ b/cmd/cloudexec/clean.go @@ -4,9 +4,9 @@ import ( "fmt" "strings" - "github.com/trailofbits/cloudexec/pkg/config" - do "github.com/trailofbits/cloudexec/pkg/digitalocean" - "github.com/trailofbits/cloudexec/pkg/s3" + "github.com/crytic/cloudexec/pkg/config" + do "github.com/crytic/cloudexec/pkg/digitalocean" + "github.com/crytic/cloudexec/pkg/s3" ) func ConfirmDeleteDroplets(config config.Config, userName string, instanceToJobs map[int64][]int64) error { diff --git a/cmd/cloudexec/configure.go b/cmd/cloudexec/configure.go index bc0cf17..d800e22 100644 --- a/cmd/cloudexec/configure.go +++ b/cmd/cloudexec/configure.go @@ -8,7 +8,7 @@ import ( "os/exec" "strings" - "github.com/trailofbits/cloudexec/pkg/config" + "github.com/crytic/cloudexec/pkg/config" "golang.org/x/term" ) diff --git a/cmd/cloudexec/init.go b/cmd/cloudexec/init.go index f60049a..49988eb 100644 --- a/cmd/cloudexec/init.go +++ b/cmd/cloudexec/init.go @@ -3,8 +3,8 @@ package main import ( "fmt" - "github.com/trailofbits/cloudexec/pkg/config" - "github.com/trailofbits/cloudexec/pkg/s3" + "github.com/crytic/cloudexec/pkg/config" + "github.com/crytic/cloudexec/pkg/s3" ) func Init(bucketName string, config config.Config) error { diff --git a/cmd/cloudexec/launch.go b/cmd/cloudexec/launch.go index 0bcba1c..96670a0 100644 --- a/cmd/cloudexec/launch.go +++ b/cmd/cloudexec/launch.go @@ -9,11 +9,10 @@ import ( "time" "github.com/BurntSushi/toml" - "github.com/trailofbits/cloudexec/pkg/config" - do "github.com/trailofbits/cloudexec/pkg/digitalocean" - "github.com/trailofbits/cloudexec/pkg/s3" - "github.com/trailofbits/cloudexec/pkg/ssh" - "github.com/trailofbits/cloudexec/pkg/state" + "github.com/crytic/cloudexec/pkg/config" + do "github.com/crytic/cloudexec/pkg/digitalocean" + "github.com/crytic/cloudexec/pkg/ssh" + "github.com/crytic/cloudexec/pkg/state" ) type Commands struct { @@ -89,12 +88,6 @@ func Launch(user *user.User, config config.Config, dropletSize string, dropletRe username := user.Username bucketName := fmt.Sprintf("cloudexec-%s-trailofbits", username) - fmt.Printf("Getting or creating new bucket for %s...\n", username) - err := s3.GetOrCreateBucket(config, bucketName) - if err != nil { - return fmt.Errorf("Failed to get bucket for %s: %w", username, err) - } - // get existing state from bucket fmt.Printf("Getting existing state from bucket %s...\n", bucketName) existingState, err := state.GetState(config, bucketName) diff --git a/cmd/cloudexec/logs.go b/cmd/cloudexec/logs.go index 9583d43..97872cf 100644 --- a/cmd/cloudexec/logs.go +++ b/cmd/cloudexec/logs.go @@ -6,8 +6,8 @@ import ( "os/exec" "strings" - "github.com/trailofbits/cloudexec/pkg/config" - "github.com/trailofbits/cloudexec/pkg/s3" + "github.com/crytic/cloudexec/pkg/config" + "github.com/crytic/cloudexec/pkg/s3" ) func GetLogsFromBucket(config config.Config, jobID int, bucketName string) error { diff --git a/cmd/cloudexec/main.go b/cmd/cloudexec/main.go index cb0f34c..c6770ca 100644 --- a/cmd/cloudexec/main.go +++ b/cmd/cloudexec/main.go @@ -9,10 +9,10 @@ import ( "strconv" "time" + do "github.com/crytic/cloudexec/pkg/digitalocean" + "github.com/crytic/cloudexec/pkg/ssh" + "github.com/crytic/cloudexec/pkg/state" "github.com/olekukonko/tablewriter" - do "github.com/trailofbits/cloudexec/pkg/digitalocean" - "github.com/trailofbits/cloudexec/pkg/ssh" - "github.com/trailofbits/cloudexec/pkg/state" "github.com/urfave/cli/v2" ) @@ -39,7 +39,7 @@ func main() { app := &cli.App{ Name: "cloudexec", - Usage: "easily run cloud based fuzzing jobs", + Usage: "easily run cloud based jobs", Commands: []*cli.Command{ { Name: "check", @@ -64,23 +64,6 @@ func main() { return nil }, }, - { - Name: "init", - Usage: "Initialize a cloud fuzzing environment", - Aliases: []string{"i"}, - Action: func(*cli.Context) error { - // Abort on configuration error - if configErr != nil { - return configErr - } - - err = Init(bucketName, config) - if err != nil { - return err - } - return nil - }, - }, { Name: "configure", Usage: "Configure credentials", @@ -94,7 +77,7 @@ func main() { }, { Name: "launch", - Usage: "Launch a droplet and start a fuzzing job", + Usage: "Launch a droplet and start a job", Aliases: []string{"l"}, Flags: []cli.Flag{ &cli.StringFlag{ @@ -149,6 +132,12 @@ func main() { dropletSize := c.String("size") dropletRegion := c.String("region") + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + fmt.Printf("Launching a %s droplet in the %s region\n", dropletSize, dropletRegion) err = Launch(user, config, dropletSize, dropletRegion, lc) if err != nil { @@ -173,6 +162,12 @@ func main() { return configErr } + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + existingState, err := state.GetState(config, bucketName) if err != nil { return err @@ -207,6 +202,12 @@ func main() { return configErr } + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + instanceToJobs, err := state.GetJobIdsByInstance(config, bucketName) if err != nil { return err @@ -245,6 +246,12 @@ func main() { return configErr } + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + instanceToJobs, err := state.GetJobIdsByInstance(config, bucketName) if err != nil { return err @@ -268,7 +275,7 @@ func main() { }, { Name: "pull", - Usage: "Pulls down the results of the latest successful fuzzing job", + Usage: "Pulls down the results of the latest successful job", Flags: []cli.Flag{ &cli.IntFlag{ Name: "job", @@ -288,6 +295,12 @@ func main() { return configErr } + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + existingState, err := state.GetState(config, bucketName) if err != nil { return err @@ -330,6 +343,12 @@ func main() { return configErr } + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + existingState, err := state.GetState(config, bucketName) if err != nil { return err @@ -393,6 +412,12 @@ func main() { return configErr } + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + // Retrieve existing state existingState, err := state.GetState(config, bucketName) if err != nil { @@ -415,6 +440,12 @@ func main() { return configErr } + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + jobID := c.Args().First() // Get the job ID from the arguments if jobID == "" { fmt.Println("Please provide a job ID to remove") @@ -450,6 +481,12 @@ func main() { return configErr } + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + // Retrieve existing state existingState, err := state.GetState(config, bucketName) if err != nil { @@ -475,6 +512,13 @@ func main() { if configErr != nil { return configErr } + + // Initialize the s3 state + err = Init(bucketName, config) + if err != nil { + return err + } + // First check if there's a running job existingState, err := state.GetState(config, bucketName) if err != nil { diff --git a/cmd/cloudexec/pull.go b/cmd/cloudexec/pull.go index a015ddb..c1dc274 100644 --- a/cmd/cloudexec/pull.go +++ b/cmd/cloudexec/pull.go @@ -6,8 +6,8 @@ import ( "path/filepath" "strings" - "github.com/trailofbits/cloudexec/pkg/config" - "github.com/trailofbits/cloudexec/pkg/s3" + "github.com/crytic/cloudexec/pkg/config" + "github.com/crytic/cloudexec/pkg/s3" ) func DownloadJobOutput(config config.Config, jobID int, localPath string, bucketName string) error { diff --git a/cmd/cloudexec/push.go b/cmd/cloudexec/push.go index 2cdcb5b..aec4018 100644 --- a/cmd/cloudexec/push.go +++ b/cmd/cloudexec/push.go @@ -5,8 +5,8 @@ import ( "os" "path/filepath" - "github.com/trailofbits/cloudexec/pkg/config" - "github.com/trailofbits/cloudexec/pkg/s3" + "github.com/crytic/cloudexec/pkg/config" + "github.com/crytic/cloudexec/pkg/s3" ) func UploadDirectoryToSpaces(config config.Config, bucketName string, sourcePath string, destPath string) error { diff --git a/cmd/cloudexec/user_data.go b/cmd/cloudexec/user_data.go index 6f582f1..9839780 100644 --- a/cmd/cloudexec/user_data.go +++ b/cmd/cloudexec/user_data.go @@ -7,7 +7,7 @@ import ( "text/template" "time" - "github.com/trailofbits/cloudexec/pkg/config" + "github.com/crytic/cloudexec/pkg/config" ) type UserData struct { diff --git a/cmd/cloudexec/user_data_test.go b/cmd/cloudexec/user_data_test.go index b00b2eb..b5210ce 100644 --- a/cmd/cloudexec/user_data_test.go +++ b/cmd/cloudexec/user_data_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/trailofbits/cloudexec/pkg/config" + "github.com/crytic/cloudexec/pkg/config" ) func getLaunchConfig(duration string) LaunchConfig { From 036e94243c459334357c1723547c0b72340964cb Mon Sep 17 00:00:00 2001 From: bohendo Date: Tue, 19 Sep 2023 17:34:59 -0400 Subject: [PATCH 03/22] move get-or-create logic into cmd/init --- cmd/cloudexec/init.go | 82 +++++++++++++++++++++++++++++++++++-------- cmd/cloudexec/main.go | 20 +++++------ pkg/s3/s3.go | 73 +++++++++++++++++--------------------- 3 files changed, 109 insertions(+), 66 deletions(-) diff --git a/cmd/cloudexec/init.go b/cmd/cloudexec/init.go index 49988eb..a07f9da 100644 --- a/cmd/cloudexec/init.go +++ b/cmd/cloudexec/init.go @@ -7,25 +7,77 @@ import ( "github.com/crytic/cloudexec/pkg/s3" ) -func Init(bucketName string, config config.Config) error { - // Create a new bucket (or get an existing one) - err := s3.GetOrCreateBucket(config, bucketName) +func Init(config config.Config, bucket string) error { + // Get a list of existing buckets + listBucketsOutput, err := s3.ListBuckets(config) if err != nil { - return fmt.Errorf("Failed to get %s bucket: %w", bucketName, err) + return fmt.Errorf("Failed to list buckets: %w", err) } - fmt.Printf("Using bucket: %v\n", bucketName) - // Create the state directory - err = s3.PutObject(config, bucketName, "state/", []byte{}) - if err != nil { - return fmt.Errorf("Failed to create state directory in bucket %s: %w", bucketName, err) - } + // Return if the desired bucket already exists + bucketExists := false + for _, thisBucket := range listBucketsOutput { + if thisBucket == bucket { + bucketExists = true + } + } - // Create the initial state file - err = s3.PutObject(config, bucketName, "state/state.json", []byte("{}")) - if err != nil { - return fmt.Errorf("Failed to create state file in bucket %s: %w", bucketName, err) - } + if !bucketExists { + // Create a new bucket + fmt.Printf("Creating new %s bucket...\n", bucket) + err = s3.CreateBucket(config, bucket) + if err != nil { + return fmt.Errorf("Failed to get %s bucket: %w", bucket, err) + } + } else { + fmt.Printf("Using existing bucket %s\n", bucket) + } + + // Ensure versioning is enabled, necessary if bucket creation was interrupted + err = s3.SetVersioning(config, bucket) + if err != nil { + return err + } + + // Initialize bucket state if not already present + err = initState(config, bucket) + if err != nil { + return fmt.Errorf("Failed to initialize state for bucket %s: %w", bucket, err) + } + + fmt.Printf("Initialized bucket %s\n", bucket) + return nil +} + +func initState(config config.Config, bucket string) error { + // Check if the state directory already exists + stateDir := "state/" + stateDirExists, err := s3.ObjectExists(config, bucket, stateDir) + if err != nil { + return fmt.Errorf("Failed to check whether the state directory exists: %w", err) + } + // Create the state directory if it does not already exist + if !stateDirExists { + fmt.Printf("Creating new state directory at %s/%s", bucket, stateDir) + err = s3.PutObject(config, bucket, stateDir, []byte{}) + if err != nil { + return fmt.Errorf("Failed to create state directory at %s/%s: %w", bucket, stateDir, err) + } + } + + // Check if the state file already exists + statePath := "state/state.json" + statePathExists, err := s3.ObjectExists(config, bucket, statePath) + if err != nil { + return fmt.Errorf("Failed to check whether the state file exists: %w", err) + } + // Create the initial state file if it does not already exist + if !statePathExists { + err = s3.PutObject(config, bucket, statePath, []byte("{}")) + if err != nil { + return fmt.Errorf("Failed to create state file in bucket %s: %w", bucket, err) + } + } return nil } diff --git a/cmd/cloudexec/main.go b/cmd/cloudexec/main.go index c6770ca..8668399 100644 --- a/cmd/cloudexec/main.go +++ b/cmd/cloudexec/main.go @@ -133,7 +133,7 @@ func main() { dropletRegion := c.String("region") // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } @@ -163,7 +163,7 @@ func main() { } // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } @@ -203,7 +203,7 @@ func main() { } // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } @@ -247,7 +247,7 @@ func main() { } // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } @@ -296,7 +296,7 @@ func main() { } // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } @@ -344,7 +344,7 @@ func main() { } // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } @@ -413,7 +413,7 @@ func main() { } // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } @@ -441,7 +441,7 @@ func main() { } // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } @@ -482,7 +482,7 @@ func main() { } // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } @@ -514,7 +514,7 @@ func main() { } // Initialize the s3 state - err = Init(bucketName, config) + err = Init(config, bucketName) if err != nil { return err } diff --git a/pkg/s3/s3.go b/pkg/s3/s3.go index 894cc98..139a7fa 100644 --- a/pkg/s3/s3.go +++ b/pkg/s3/s3.go @@ -95,35 +95,26 @@ func ListBuckets(config config.Config) ([]string, error) { return buckets, nil } -func GetOrCreateBucket(config config.Config, bucket string) error { - listBucketsOutput, err := ListBuckets(config) - if err != nil { - return fmt.Errorf("Failed to list buckets: %w", err) - } - - // Check if the desired Space already exists - for _, thisBucket := range listBucketsOutput { - if thisBucket == bucket { - // create a non-init client - s3Client, err := initializeS3Client(config, false) - if err != nil { - return err - } - - // ensure versioning is enabled on the bucket - _, err = s3Client.PutBucketVersioning(&s3.PutBucketVersioningInput{ - Bucket: aws.String(bucket), - VersioningConfiguration: &s3.VersioningConfiguration{ - Status: aws.String("Enabled"), - }, - }) - if err != nil { - return fmt.Errorf("Failed to enable versioning on bucket '%s': %w", bucket, err) - } - return nil - } - } +func SetVersioning(config config.Config, bucket string) error { + // create a non-init client + s3Client, err := initializeS3Client(config, false) + if err != nil { + return err + } + // ensure versioning is enabled on the bucket + _, err = s3Client.PutBucketVersioning(&s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + }) + if err != nil { + return fmt.Errorf("Failed to enable versioning on bucket '%s': %w", bucket, err) + } + return nil +} +func CreateBucket(config config.Config, bucket string) error { // create an initialization client s3Client, err := initializeS3Client(config, true) if err != nil { @@ -149,21 +140,11 @@ func GetOrCreateBucket(config config.Config, bucket string) error { return fmt.Errorf("Failed to wait for bucket '%s': %w", bucket, err) } - // enable versioning on the bucket - _, err = s3Client.PutBucketVersioning(&s3.PutBucketVersioningInput{ - Bucket: aws.String(bucket), - VersioningConfiguration: &s3.VersioningConfiguration{ - Status: aws.String("Enabled"), - }, - }) - if err != nil { - return fmt.Errorf("Failed to enable versioning on bucket '%s': %w", bucket, err) - } - fmt.Printf("Created bucket '%s'...\n", bucket) return nil } +// Note: will overwrite existing object. func PutObject(config config.Config, bucket string, key string, value []byte) error { // create a client s3Client, err := initializeS3Client(config, false) @@ -173,10 +154,9 @@ func PutObject(config config.Config, bucket string, key string, value []byte) er // If zero-length value is given, create a directory instead of a file if len(value) == 0 { - // Create the state directory _, err = s3Client.PutObject(&s3.PutObjectInput{ Bucket: aws.String(bucket), - Key: aws.String("state/"), + Key: aws.String(key), }) if err != nil { return fmt.Errorf("Failed to create %s directory in bucket %s: %w", key, bucket, err) @@ -206,6 +186,17 @@ func PutObject(config config.Config, bucket string, key string, value []byte) er return nil } +func ObjectExists(config config.Config, bucket string, key string) (bool, error) { + // Get a list of objects that are prefixed by the target key + objects, err := ListObjects(config, bucket, key) + if err != nil { + return false, err + } + + // return true if we got a non-zero number of objects that match the prefix + return len(objects) != 0, nil +} + func GetObject(config config.Config, bucket string, key string) ([]byte, error) { // create a client s3Client, err := initializeS3Client(config, false) From 071be985a2b03934661ade4e0379eb99245e3147 Mon Sep 17 00:00:00 2001 From: bohendo Date: Tue, 19 Sep 2023 18:02:02 -0400 Subject: [PATCH 04/22] allow cleanup even if no jobs present --- pkg/state/state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/state/state.go b/pkg/state/state.go index db2f85b..75ac687 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -205,10 +205,10 @@ func GetJobIdsByInstance(config config.Config, bucketName string) (map[int64][]i if err != nil { return nil, fmt.Errorf("Failed to get state: %w", err) } + instanceToJobIds := make(map[int64][]int64) if existingState.Jobs == nil { - return nil, fmt.Errorf("No jobs found in the existing state") + return instanceToJobIds, nil } - instanceToJobIds := make(map[int64][]int64) for _, job := range existingState.Jobs { instanceToJobIds[job.InstanceID] = append(instanceToJobIds[job.InstanceID], job.ID) } From e529179d1de60dd6379e6deee53f7be9a3ad3751 Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 20 Sep 2023 13:52:34 -0400 Subject: [PATCH 05/22] rename release token --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 21ee2bb..340b0ab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,4 +53,4 @@ jobs: version: latest args: release --clean --skip-validate env: - GITHUB_TOKEN: ${{ secrets.OLDSJ_GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_RELEASE_TOKEN }} From 2619edc7f498894573b97a88a114b7044bc364f1 Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 20 Sep 2023 14:02:21 -0400 Subject: [PATCH 06/22] improve log formatting --- cmd/cloudexec/init.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/cloudexec/init.go b/cmd/cloudexec/init.go index a07f9da..db3af20 100644 --- a/cmd/cloudexec/init.go +++ b/cmd/cloudexec/init.go @@ -29,8 +29,6 @@ func Init(config config.Config, bucket string) error { if err != nil { return fmt.Errorf("Failed to get %s bucket: %w", bucket, err) } - } else { - fmt.Printf("Using existing bucket %s\n", bucket) } // Ensure versioning is enabled, necessary if bucket creation was interrupted @@ -45,7 +43,6 @@ func Init(config config.Config, bucket string) error { return fmt.Errorf("Failed to initialize state for bucket %s: %w", bucket, err) } - fmt.Printf("Initialized bucket %s\n", bucket) return nil } @@ -58,7 +55,7 @@ func initState(config config.Config, bucket string) error { } // Create the state directory if it does not already exist if !stateDirExists { - fmt.Printf("Creating new state directory at %s/%s", bucket, stateDir) + fmt.Printf("Creating new state directory at %s/%s\n", bucket, stateDir) err = s3.PutObject(config, bucket, stateDir, []byte{}) if err != nil { return fmt.Errorf("Failed to create state directory at %s/%s: %w", bucket, stateDir, err) @@ -73,6 +70,7 @@ func initState(config config.Config, bucket string) error { } // Create the initial state file if it does not already exist if !statePathExists { + fmt.Printf("Creating new state file at %s/%s\n", bucket, statePath) err = s3.PutObject(config, bucket, statePath, []byte("{}")) if err != nil { return fmt.Errorf("Failed to create state file in bucket %s: %w", bucket, err) From 29d775dbcf90cb84f5f84334d299fd645d203410 Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 20 Sep 2023 15:22:43 -0400 Subject: [PATCH 07/22] escape double quotes in run/setup commands --- cmd/cloudexec/user_data.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/cloudexec/user_data.go b/cmd/cloudexec/user_data.go index 9839780..cf05fb2 100644 --- a/cmd/cloudexec/user_data.go +++ b/cmd/cloudexec/user_data.go @@ -4,6 +4,7 @@ import ( "bytes" _ "embed" "fmt" + "strings" "text/template" "time" @@ -36,13 +37,14 @@ func GenerateUserData(config config.Config, lc LaunchConfig) (string, error) { timeoutStr := fmt.Sprintf("%d", int(timeout.Seconds())) // Set the values for the template + // double quotes are escaped so the command strings can be safely contained by double quotes in bash data := UserData{ SpacesAccessKey: config.DigitalOcean.SpacesAccessKey, SpacesSecretKey: config.DigitalOcean.SpacesSecretKey, SpacesRegion: config.DigitalOcean.SpacesRegion, DigitalOceanToken: config.DigitalOcean.ApiKey, - SetupCommands: lc.Commands.Setup, - RunCommand: lc.Commands.Run, + SetupCommands: strings.ReplaceAll(lc.Commands.Setup, `"`, `\"`), + RunCommand: strings.ReplaceAll(lc.Commands.Run, `"`, `\"`), Timeout: timeoutStr, } From 7592bef9e744fff16cabc3a08237382ea31ae0ed Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 20 Sep 2023 16:02:56 -0400 Subject: [PATCH 08/22] run setup script before accessing state --- cmd/cloudexec/user_data.sh.tmpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/cloudexec/user_data.sh.tmpl b/cmd/cloudexec/user_data.sh.tmpl index 46291ff..dd7ff99 100644 --- a/cmd/cloudexec/user_data.sh.tmpl +++ b/cmd/cloudexec/user_data.sh.tmpl @@ -98,6 +98,10 @@ update_state() { # Set the trap to call the cleanup function on signals or exit trap cleanup EXIT SIGHUP SIGINT SIGTERM +echo "Running setup..." +mkdir -p ~/output +eval "${SETUP_COMMANDS}" + echo "Setting up DigitalOcean credentials..." # ensure these are set in the environment if [[ -z ${DIGITALOCEAN_ACCESS_TOKEN} ]]; then @@ -152,10 +156,6 @@ fi source ~/venv/bin/activate -echo "Running setup..." -mkdir -p ~/output -eval "${SETUP_COMMANDS}" - # Update state to running update_state "running" From fde78dae6e75b0bc044a572e529a222c7cdb1b62 Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 20 Sep 2023 16:23:41 -0400 Subject: [PATCH 09/22] use default image instead of failing if no snapshots --- pkg/digitalocean/digitalocean.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/digitalocean/digitalocean.go b/pkg/digitalocean/digitalocean.go index ee9c227..391d2ba 100644 --- a/pkg/digitalocean/digitalocean.go +++ b/pkg/digitalocean/digitalocean.go @@ -301,8 +301,11 @@ func GetLatestSnapshot(config config.Config) (Snapshot, error) { options.Page++ } - if latestSnapshot == nil { - return empty, fmt.Errorf("Failed to find cloudexec snapshot") + if latestSnapshot == nil { + return Snapshot{ + ID: "ubuntu-22-04-x64", + Name: "default", + }, nil } return Snapshot{ From e75e20e2e748241bc25ade658303e3874e9f2bca Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 20 Sep 2023 16:42:57 -0400 Subject: [PATCH 10/22] move provider.sh stuff into example setup script --- example/cloudexec.toml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/example/cloudexec.toml b/example/cloudexec.toml index e629c8d..7bfcdb1 100644 --- a/example/cloudexec.toml +++ b/example/cloudexec.toml @@ -4,6 +4,42 @@ timeout = "48h" [commands] setup = ''' +# set hostname +echo "Setting hostname..." +echo "cloudexec" >/etc/hostname +hostname -F /etc/hostname + +echo "Installing prereqs..." +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y jq s3cmd tmux python3-pip python3-venv unzip + +echo "Downloading doctl..." +curl -fsSL -o /tmp/doctl-1.92.0-linux-amd64.tar.gz https://github.com/digitalocean/doctl/releases/download/v1.92.0/doctl-1.92.0-linux-amd64.tar.gz +echo "Extracting doctl..." +tar -xzf /tmp/doctl-1.92.0-linux-amd64.tar.gz -C /tmp +echo "Installing doctl..." +mv /tmp/doctl /usr/local/bin +echo "Cleaning up..." +rm /tmp/doctl-1.92.0-linux-amd64.tar.gz + +echo "Installing solc and slither..." +python3 -m venv ~/venv +source ~/venv/bin/activate +pip3 install solc-select slither-analyzer crytic-compile +solc-select install 0.8.6 +solc-select use 0.8.6 + +echo "Downloading echidna..." +curl -fsSL -o /tmp/echidna.zip https://github.com/crytic/echidna/releases/download/v2.2.1/echidna-2.2.1-Linux.zip +echo "Extracting echidna..." +unzip /tmp/echidna.zip -d /tmp +tar -xzf /tmp/echidna.tar.gz -C /tmp +echo "Installing echidna..." +mv /tmp/echidna /usr/local/bin +rm /tmp/echidna.tar.gz + +echo "Downloading medusa..." sudo apt-get update; sudo apt-get install -y unzip curl -fsSL https://github.com/crytic/medusa/releases/download/v0.1.0/medusa-linux-x64.zip -o medusa.zip unzip medusa.zip From 41243e7531c01dfab6fcd992cd9bd79eadd87a0d Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 20 Sep 2023 17:04:10 -0400 Subject: [PATCH 11/22] sync example setup and provision.sh --- example/cloudexec.toml | 19 +++++++++++++++++++ packer/provision.sh | 25 +++++++++++++++++++------ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/example/cloudexec.toml b/example/cloudexec.toml index 7bfcdb1..82e7683 100644 --- a/example/cloudexec.toml +++ b/example/cloudexec.toml @@ -4,6 +4,10 @@ timeout = "48h" [commands] setup = ''' + +######################################## +## Required Configuration and Dependencies + # set hostname echo "Setting hostname..." echo "cloudexec" >/etc/hostname @@ -23,6 +27,9 @@ mv /tmp/doctl /usr/local/bin echo "Cleaning up..." rm /tmp/doctl-1.92.0-linux-amd64.tar.gz +######################################## +## Common fuzz testing and analysis tools + echo "Installing solc and slither..." python3 -m venv ~/venv source ~/venv/bin/activate @@ -45,6 +52,18 @@ curl -fsSL https://github.com/crytic/medusa/releases/download/v0.1.0/medusa-linu unzip medusa.zip chmod +x medusa sudo mv medusa /usr/local/bin + +echo "Installing docker and its dependencies..." +apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common +docker_key="$(curl -fsSL https://download.docker.com/linux/ubuntu/gpg)" +echo "${docker_key}" | apt-key add - +release="$(lsb_release -cs)" +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu ${release} stable" +apt-get update -y +apt-get install -y docker-ce docker-ce-cli containerd.io +user="$(whoami)" +usermod -aG docker "${user}" +systemctl enable docker ''' # This command is run after the setup script completes. diff --git a/packer/provision.sh b/packer/provision.sh index 2c24ed0..34ab42a 100644 --- a/packer/provision.sh +++ b/packer/provision.sh @@ -2,6 +2,9 @@ # shellcheck source=/dev/null set -e +######################################## +## Required Configuration and Dependencies + # set hostname echo "Setting hostname..." echo "cloudexec" >/etc/hostname @@ -21,6 +24,16 @@ mv /tmp/doctl /usr/local/bin echo "Cleaning up..." rm /tmp/doctl-1.92.0-linux-amd64.tar.gz +######################################## +## Common fuzz testing and analysis tools + +echo "Installing solc and slither..." +python3 -m venv ~/venv +source ~/venv/bin/activate +pip3 install solc-select slither-analyzer crytic-compile +solc-select install 0.8.6 +solc-select use 0.8.6 + echo "Downloading echidna..." curl -fsSL -o /tmp/echidna.zip https://github.com/crytic/echidna/releases/download/v2.2.1/echidna-2.2.1-Linux.zip echo "Extracting echidna..." @@ -30,12 +43,12 @@ echo "Installing echidna..." mv /tmp/echidna /usr/local/bin rm /tmp/echidna.tar.gz -echo "Installing solc and slither..." -python3 -m venv ~/venv -source ~/venv/bin/activate -pip3 install solc-select slither-analyzer crytic-compile -solc-select install 0.8.6 -solc-select use 0.8.6 +echo "Downloading medusa..." +sudo apt-get update; sudo apt-get install -y unzip +curl -fsSL https://github.com/crytic/medusa/releases/download/v0.1.0/medusa-linux-x64.zip -o medusa.zip +unzip medusa.zip +chmod +x medusa +sudo mv medusa /usr/local/bin echo "Installing docker and its dependencies..." apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common From a2dfba75a2478aa3b44ede6f09fb3c5e9f4b9437 Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 20 Sep 2023 17:34:37 -0400 Subject: [PATCH 12/22] document packer --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 561b874..09afdf0 100644 --- a/README.md +++ b/README.md @@ -189,3 +189,17 @@ cloudexec clean ``` Note that there is often a delay while deleting files from Digital Ocean Spaces buckets. + +## Optional: Create a cloudexec DigitalOcean image + +Building and uploading a dedicated DigitalOcean image for cloudfuzz will simplify your launch configuration and improve startup times. + +To do so, install `packer` with `brew install packer`. If you're using `nix` and `direnv`, it's added to your PATH via the flake's dev shell. + +To build and upload a docker image, run the following command. Make sure your DigitalOcean API key is either in your env vars or replace it with the actual token. + +`packer build -var do_api_token=$DIGITALOCEAN_API_KEY cloudexec.pkr.hcl` + +This will take care of everything and if you visit the [DigitalOcean snapshots page](https://cloud.digitalocean.com/images/snapshots/droplets), you'll see a snapshot called `cloudexec-20230920164605` or similar. `cloudexec` will search for snapshots starts with a `cloudexec-` prefix and it will use the one with the most recent timestamp string. + +Now, you can remove everything from the setup command in the example launch config or replace it to install additional tools. From 44e27e3f7a2051895cb422a94c946c4ec16b2e3a Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 20 Sep 2023 17:52:04 -0400 Subject: [PATCH 13/22] trunk fmt --- README.md | 2 +- cmd/cloudexec/init.go | 102 +++++++++++++++---------------- cmd/cloudexec/main.go | 54 ++++++++-------- cmd/cloudexec/user_data.go | 4 +- packer/provision.sh | 3 +- pkg/digitalocean/digitalocean.go | 10 +-- pkg/s3/s3.go | 40 ++++++------ pkg/state/state.go | 2 +- 8 files changed, 109 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index 09afdf0..84c5644 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ cloudexec clean Note that there is often a delay while deleting files from Digital Ocean Spaces buckets. ## Optional: Create a cloudexec DigitalOcean image - + Building and uploading a dedicated DigitalOcean image for cloudfuzz will simplify your launch configuration and improve startup times. To do so, install `packer` with `brew install packer`. If you're using `nix` and `direnv`, it's added to your PATH via the flake's dev shell. diff --git a/cmd/cloudexec/init.go b/cmd/cloudexec/init.go index db3af20..3699eba 100644 --- a/cmd/cloudexec/init.go +++ b/cmd/cloudexec/init.go @@ -8,74 +8,74 @@ import ( ) func Init(config config.Config, bucket string) error { - // Get a list of existing buckets + // Get a list of existing buckets listBucketsOutput, err := s3.ListBuckets(config) if err != nil { return fmt.Errorf("Failed to list buckets: %w", err) } // Return if the desired bucket already exists - bucketExists := false + bucketExists := false for _, thisBucket := range listBucketsOutput { if thisBucket == bucket { - bucketExists = true - } - } + bucketExists = true + } + } - if !bucketExists { - // Create a new bucket - fmt.Printf("Creating new %s bucket...\n", bucket) - err = s3.CreateBucket(config, bucket) - if err != nil { - return fmt.Errorf("Failed to get %s bucket: %w", bucket, err) - } - } + if !bucketExists { + // Create a new bucket + fmt.Printf("Creating new %s bucket...\n", bucket) + err = s3.CreateBucket(config, bucket) + if err != nil { + return fmt.Errorf("Failed to get %s bucket: %w", bucket, err) + } + } - // Ensure versioning is enabled, necessary if bucket creation was interrupted - err = s3.SetVersioning(config, bucket) - if err != nil { - return err - } + // Ensure versioning is enabled, necessary if bucket creation was interrupted + err = s3.SetVersioning(config, bucket) + if err != nil { + return err + } - // Initialize bucket state if not already present - err = initState(config, bucket) - if err != nil { - return fmt.Errorf("Failed to initialize state for bucket %s: %w", bucket, err) - } + // Initialize bucket state if not already present + err = initState(config, bucket) + if err != nil { + return fmt.Errorf("Failed to initialize state for bucket %s: %w", bucket, err) + } return nil } func initState(config config.Config, bucket string) error { - // Check if the state directory already exists - stateDir := "state/" - stateDirExists, err := s3.ObjectExists(config, bucket, stateDir) - if err != nil { - return fmt.Errorf("Failed to check whether the state directory exists: %w", err) - } - // Create the state directory if it does not already exist - if !stateDirExists { - fmt.Printf("Creating new state directory at %s/%s\n", bucket, stateDir) - err = s3.PutObject(config, bucket, stateDir, []byte{}) - if err != nil { - return fmt.Errorf("Failed to create state directory at %s/%s: %w", bucket, stateDir, err) - } - } + // Check if the state directory already exists + stateDir := "state/" + stateDirExists, err := s3.ObjectExists(config, bucket, stateDir) + if err != nil { + return fmt.Errorf("Failed to check whether the state directory exists: %w", err) + } + // Create the state directory if it does not already exist + if !stateDirExists { + fmt.Printf("Creating new state directory at %s/%s\n", bucket, stateDir) + err = s3.PutObject(config, bucket, stateDir, []byte{}) + if err != nil { + return fmt.Errorf("Failed to create state directory at %s/%s: %w", bucket, stateDir, err) + } + } - // Check if the state file already exists - statePath := "state/state.json" - statePathExists, err := s3.ObjectExists(config, bucket, statePath) - if err != nil { - return fmt.Errorf("Failed to check whether the state file exists: %w", err) - } - // Create the initial state file if it does not already exist - if !statePathExists { - fmt.Printf("Creating new state file at %s/%s\n", bucket, statePath) - err = s3.PutObject(config, bucket, statePath, []byte("{}")) - if err != nil { - return fmt.Errorf("Failed to create state file in bucket %s: %w", bucket, err) - } - } + // Check if the state file already exists + statePath := "state/state.json" + statePathExists, err := s3.ObjectExists(config, bucket, statePath) + if err != nil { + return fmt.Errorf("Failed to check whether the state file exists: %w", err) + } + // Create the initial state file if it does not already exist + if !statePathExists { + fmt.Printf("Creating new state file at %s/%s\n", bucket, statePath) + err = s3.PutObject(config, bucket, statePath, []byte("{}")) + if err != nil { + return fmt.Errorf("Failed to create state file in bucket %s: %w", bucket, err) + } + } return nil } diff --git a/cmd/cloudexec/main.go b/cmd/cloudexec/main.go index 8668399..f139a68 100644 --- a/cmd/cloudexec/main.go +++ b/cmd/cloudexec/main.go @@ -31,7 +31,7 @@ func main() { os.Exit(1) } userName := user.Username - // TODO: sanitize username usage in bucketname + // TODO: sanitize username usage in bucketname bucketName := fmt.Sprintf("cloudexec-%s-trailofbits", userName) // Attempt to load the configuration @@ -132,7 +132,7 @@ func main() { dropletSize := c.String("size") dropletRegion := c.String("region") - // Initialize the s3 state + // Initialize the s3 state err = Init(config, bucketName) if err != nil { return err @@ -162,7 +162,7 @@ func main() { return configErr } - // Initialize the s3 state + // Initialize the s3 state err = Init(config, bucketName) if err != nil { return err @@ -202,7 +202,7 @@ func main() { return configErr } - // Initialize the s3 state + // Initialize the s3 state err = Init(config, bucketName) if err != nil { return err @@ -246,7 +246,7 @@ func main() { return configErr } - // Initialize the s3 state + // Initialize the s3 state err = Init(config, bucketName) if err != nil { return err @@ -295,7 +295,7 @@ func main() { return configErr } - // Initialize the s3 state + // Initialize the s3 state err = Init(config, bucketName) if err != nil { return err @@ -343,7 +343,7 @@ func main() { return configErr } - // Initialize the s3 state + // Initialize the s3 state err = Init(config, bucketName) if err != nil { return err @@ -412,11 +412,11 @@ func main() { return configErr } - // Initialize the s3 state - err = Init(config, bucketName) - if err != nil { - return err - } + // Initialize the s3 state + err = Init(config, bucketName) + if err != nil { + return err + } // Retrieve existing state existingState, err := state.GetState(config, bucketName) @@ -440,11 +440,11 @@ func main() { return configErr } - // Initialize the s3 state - err = Init(config, bucketName) - if err != nil { - return err - } + // Initialize the s3 state + err = Init(config, bucketName) + if err != nil { + return err + } jobID := c.Args().First() // Get the job ID from the arguments if jobID == "" { @@ -481,11 +481,11 @@ func main() { return configErr } - // Initialize the s3 state - err = Init(config, bucketName) - if err != nil { - return err - } + // Initialize the s3 state + err = Init(config, bucketName) + if err != nil { + return err + } // Retrieve existing state existingState, err := state.GetState(config, bucketName) @@ -513,11 +513,11 @@ func main() { return configErr } - // Initialize the s3 state - err = Init(config, bucketName) - if err != nil { - return err - } + // Initialize the s3 state + err = Init(config, bucketName) + if err != nil { + return err + } // First check if there's a running job existingState, err := state.GetState(config, bucketName) diff --git a/cmd/cloudexec/user_data.go b/cmd/cloudexec/user_data.go index cf05fb2..c8d058e 100644 --- a/cmd/cloudexec/user_data.go +++ b/cmd/cloudexec/user_data.go @@ -37,14 +37,14 @@ func GenerateUserData(config config.Config, lc LaunchConfig) (string, error) { timeoutStr := fmt.Sprintf("%d", int(timeout.Seconds())) // Set the values for the template - // double quotes are escaped so the command strings can be safely contained by double quotes in bash + // double quotes are escaped so the command strings can be safely contained by double quotes in bash data := UserData{ SpacesAccessKey: config.DigitalOcean.SpacesAccessKey, SpacesSecretKey: config.DigitalOcean.SpacesSecretKey, SpacesRegion: config.DigitalOcean.SpacesRegion, DigitalOceanToken: config.DigitalOcean.ApiKey, SetupCommands: strings.ReplaceAll(lc.Commands.Setup, `"`, `\"`), - RunCommand: strings.ReplaceAll(lc.Commands.Run, `"`, `\"`), + RunCommand: strings.ReplaceAll(lc.Commands.Run, `"`, `\"`), Timeout: timeoutStr, } diff --git a/packer/provision.sh b/packer/provision.sh index 34ab42a..ba992b4 100644 --- a/packer/provision.sh +++ b/packer/provision.sh @@ -44,7 +44,8 @@ mv /tmp/echidna /usr/local/bin rm /tmp/echidna.tar.gz echo "Downloading medusa..." -sudo apt-get update; sudo apt-get install -y unzip +sudo apt-get update +sudo apt-get install -y unzip curl -fsSL https://github.com/crytic/medusa/releases/download/v0.1.0/medusa-linux-x64.zip -o medusa.zip unzip medusa.zip chmod +x medusa diff --git a/pkg/digitalocean/digitalocean.go b/pkg/digitalocean/digitalocean.go index 391d2ba..fa987e0 100644 --- a/pkg/digitalocean/digitalocean.go +++ b/pkg/digitalocean/digitalocean.go @@ -301,11 +301,11 @@ func GetLatestSnapshot(config config.Config) (Snapshot, error) { options.Page++ } - if latestSnapshot == nil { - return Snapshot{ - ID: "ubuntu-22-04-x64", - Name: "default", - }, nil + if latestSnapshot == nil { + return Snapshot{ + ID: "ubuntu-22-04-x64", + Name: "default", + }, nil } return Snapshot{ diff --git a/pkg/s3/s3.go b/pkg/s3/s3.go index 0855274..6b26eb9 100644 --- a/pkg/s3/s3.go +++ b/pkg/s3/s3.go @@ -96,22 +96,22 @@ func ListBuckets(config config.Config) ([]string, error) { } func SetVersioning(config config.Config, bucket string) error { - // create a non-init client - s3Client, err := initializeS3Client(config, false) - if err != nil { - return err - } - // ensure versioning is enabled on the bucket - _, err = s3Client.PutBucketVersioning(&s3.PutBucketVersioningInput{ - Bucket: aws.String(bucket), - VersioningConfiguration: &s3.VersioningConfiguration{ - Status: aws.String("Enabled"), - }, - }) - if err != nil { - return fmt.Errorf("Failed to enable versioning on bucket '%s': %w", bucket, err) - } - return nil + // create a non-init client + s3Client, err := initializeS3Client(config, false) + if err != nil { + return err + } + // ensure versioning is enabled on the bucket + _, err = s3Client.PutBucketVersioning(&s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + }) + if err != nil { + return fmt.Errorf("Failed to enable versioning on bucket '%s': %w", bucket, err) + } + return nil } func CreateBucket(config config.Config, bucket string) error { @@ -187,14 +187,14 @@ func PutObject(config config.Config, bucket string, key string, value []byte) er } func ObjectExists(config config.Config, bucket string, key string) (bool, error) { - // Get a list of objects that are prefixed by the target key - objects, err := ListObjects(config, bucket, key) + // Get a list of objects that are prefixed by the target key + objects, err := ListObjects(config, bucket, key) if err != nil { return false, err } - // return true if we got a non-zero number of objects that match the prefix - return len(objects) != 0, nil + // return true if we got a non-zero number of objects that match the prefix + return len(objects) != 0, nil } func GetObject(config config.Config, bucket string, key string) ([]byte, error) { diff --git a/pkg/state/state.go b/pkg/state/state.go index 3d98e77..abdba19 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -207,7 +207,7 @@ func GetJobIdsByInstance(config config.Config, bucketName string) (map[int64][]i } instanceToJobIds := make(map[int64][]int64) if existingState.Jobs == nil { - return instanceToJobIds, nil + return instanceToJobIds, nil } for _, job := range existingState.Jobs { instanceToJobIds[job.InstanceID] = append(instanceToJobIds[job.InstanceID], job.ID) From 68113fecf46a882f97f1e8aac972e8e74522159e Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 22 Sep 2023 09:31:31 -0400 Subject: [PATCH 14/22] rename release token secret --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 340b0ab..b01a70d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,4 +53,4 @@ jobs: version: latest args: release --clean --skip-validate env: - GITHUB_TOKEN: ${{ secrets.GITHUB_RELEASE_TOKEN }} + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} From 87465f4f9c618ad25347e66dc73fbbbfb207d2c2 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 22 Sep 2023 09:48:24 -0400 Subject: [PATCH 15/22] clean up readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 84c5644..2d72d76 100644 --- a/README.md +++ b/README.md @@ -190,9 +190,9 @@ cloudexec clean Note that there is often a delay while deleting files from Digital Ocean Spaces buckets. -## Optional: Create a cloudexec DigitalOcean image +## Optional: Create a CloudExec DigitalOcean image -Building and uploading a dedicated DigitalOcean image for cloudfuzz will simplify your launch configuration and improve startup times. +Building and uploading a dedicated DigitalOcean image for `cloudexec` will simplify your launch configuration and improve startup times. To do so, install `packer` with `brew install packer`. If you're using `nix` and `direnv`, it's added to your PATH via the flake's dev shell. From eaa08777be2fb31169197790fec8af3b5613c28e Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 22 Sep 2023 10:13:01 -0400 Subject: [PATCH 16/22] remove -trailofbits suffix from bucket name --- cmd/cloudexec/launch.go | 2 +- cmd/cloudexec/main.go | 2 +- cmd/cloudexec/user_data.sh.tmpl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/cloudexec/launch.go b/cmd/cloudexec/launch.go index 96670a0..87f85e5 100644 --- a/cmd/cloudexec/launch.go +++ b/cmd/cloudexec/launch.go @@ -86,7 +86,7 @@ func LoadLaunchConfig(launchConfigPath string) (LaunchConfig, error) { func Launch(user *user.User, config config.Config, dropletSize string, dropletRegion string, lc LaunchConfig) error { username := user.Username - bucketName := fmt.Sprintf("cloudexec-%s-trailofbits", username) + bucketName := fmt.Sprintf("cloudexec-%s", username) // get existing state from bucket fmt.Printf("Getting existing state from bucket %s...\n", bucketName) diff --git a/cmd/cloudexec/main.go b/cmd/cloudexec/main.go index f139a68..c382e5d 100644 --- a/cmd/cloudexec/main.go +++ b/cmd/cloudexec/main.go @@ -32,7 +32,7 @@ func main() { } userName := user.Username // TODO: sanitize username usage in bucketname - bucketName := fmt.Sprintf("cloudexec-%s-trailofbits", userName) + bucketName := fmt.Sprintf("cloudexec-%s", userName) // Attempt to load the configuration config, configErr := LoadConfig(configFilePath) diff --git a/cmd/cloudexec/user_data.sh.tmpl b/cmd/cloudexec/user_data.sh.tmpl index dd7ff99..1c02f79 100644 --- a/cmd/cloudexec/user_data.sh.tmpl +++ b/cmd/cloudexec/user_data.sh.tmpl @@ -135,7 +135,7 @@ if [[ ${JOB_ID} == "" ]]; then echo "No job ID, exiting..." exit 1 fi -export BUCKET_NAME="cloudexec-${USERNAME}-trailofbits" +export BUCKET_NAME="cloudexec-${USERNAME}" echo "Setting up S3 credentials..." # Spaces uses the AWS S3 API From 8a0b5714f90b609cfc79eddea66686b8d35df49b Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 22 Sep 2023 10:33:26 -0400 Subject: [PATCH 17/22] change owner in goreleaser --- .goreleaser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 3ca81b2..b48c57c 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -45,7 +45,7 @@ signs: release: github: - owner: trailofbits + owner: crytic name: cloudexec brews: From 3588da57c59be4c5674d24e2881d3a3b40205351 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 22 Sep 2023 12:59:34 -0400 Subject: [PATCH 18/22] move critical steup into the user data template script --- cmd/cloudexec/user_data.sh.tmpl | 133 ++++++++++++++++++++++---------- example/cloudexec.toml | 26 ------- 2 files changed, 91 insertions(+), 68 deletions(-) diff --git a/cmd/cloudexec/user_data.sh.tmpl b/cmd/cloudexec/user_data.sh.tmpl index 1c02f79..318b166 100644 --- a/cmd/cloudexec/user_data.sh.tmpl +++ b/cmd/cloudexec/user_data.sh.tmpl @@ -4,6 +4,9 @@ set -e shopt -s inherit_errexit +######################################## +# Setup env vars and constants + # Import env vars from user data export DIGITALOCEAN_ACCESS_TOKEN={{.DigitalOceanToken}} export AWS_ACCESS_KEY_ID={{.SpacesAccessKey}} @@ -16,6 +19,85 @@ export TIMEOUT="{{.Timeout}}" stdout_log="/tmp/cloudexec-stdout.log" stderr_log="/tmp/cloudexec-stderr.log" +######################################## +# Required setup + +echo "Installing prereqs..." +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y jq s3cmd tmux python3-pip python3-venv unzip + +# set hostname +if [[ "$(hostname)" != "cloudexec" ]] +then + echo "Setting hostname..." + echo "cloudexec" >/etc/hostname + hostname -F /etc/hostname +fi + +if ! command -v doctl >/dev/null 2>&1 +then + echo "Downloading doctl..." + curl -fsSL -o /tmp/doctl-1.92.0-linux-amd64.tar.gz https://github.com/digitalocean/doctl/releases/download/v1.92.0/doctl-1.92.0-linux-amd64.tar.gz + echo "Extracting doctl..." + tar -xzf /tmp/doctl-1.92.0-linux-amd64.tar.gz -C /tmp + echo "Installing doctl..." + mv /tmp/doctl /usr/local/bin + echo "Cleaning up..." + rm /tmp/doctl-1.92.0-linux-amd64.tar.gz +fi + +######################################## +# Confirm required env vars are present + +echo "Confirming this is a CloudExec droplet..." +TAGS=$(curl -s http://169.254.169.254/metadata/v1/tags) +echo "Droplet tags:" +echo "${TAGS}" +export JOB_ID="" +export USERNAME="" +export CLOUDEXEC=false +for tag in ${TAGS}; do + if [[ ${tag} == "Purpose:cloudexec" ]]; then + CLOUDEXEC=true + elif [[ ${tag} == "Owner:"* ]]; then + USERNAME=${tag#"Owner:"} + elif [[ ${tag} == "Job:"* ]]; then + JOB_ID=${tag#"Job:"} + fi +done + +echo "Setting up DigitalOcean credentials..." +# ensure these are set in the environment +if [[ -z ${DIGITALOCEAN_ACCESS_TOKEN} ]]; then + echo "ERROR: DIGITALOCEAN_ACCESS_TOKEN is not set" + echo "CloudExec will not be able to destroy the droplet" + echo "on exit and you will incur charges." + exit 1 +fi + +echo "Setting up S3 credentials..." +# Spaces uses the AWS S3 API +for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION; do + if [[ -z ${!var} ]]; then + echo "${var} is not set, exiting..." + exit 1 + fi +done + +if [[ ${CLOUDEXEC} == false ]] || [[ ${USERNAME} == "" ]]; then + echo "Not a CloudExec droplet, exiting..." + # exit 1 +fi + +if [[ ${JOB_ID} == "" ]]; then + echo "No job ID, exiting..." + exit 1 +fi + +######################################## +# Define helper functions + fmtDate() { date -d "@$1" "+%Y-%m-%d %H:%M:%S" } @@ -98,54 +180,15 @@ update_state() { # Set the trap to call the cleanup function on signals or exit trap cleanup EXIT SIGHUP SIGINT SIGTERM +######################################## +# Job-specific setup + echo "Running setup..." mkdir -p ~/output eval "${SETUP_COMMANDS}" -echo "Setting up DigitalOcean credentials..." -# ensure these are set in the environment -if [[ -z ${DIGITALOCEAN_ACCESS_TOKEN} ]]; then - echo "ERROR: DIGITALOCEAN_ACCESS_TOKEN is not set" - echo "CloudExec will not be able to destroy the droplet" - echo "on exit and you will incur charges." - exit 1 -fi - -echo "Confirming this is a CloudExec droplet..." -TAGS=$(curl -s http://169.254.169.254/metadata/v1/tags) -echo "Droplet tags:" -echo "${TAGS}" -export JOB_ID="" -export USERNAME="" -export CLOUDEXEC=false -for tag in ${TAGS}; do - if [[ ${tag} == "Purpose:cloudexec" ]]; then - CLOUDEXEC=true - elif [[ ${tag} == "Owner:"* ]]; then - USERNAME=${tag#"Owner:"} - elif [[ ${tag} == "Job:"* ]]; then - JOB_ID=${tag#"Job:"} - fi -done -if [[ ${CLOUDEXEC} == false ]] || [[ ${USERNAME} == "" ]]; then - echo "Not a CloudExec droplet, exiting..." - # exit 1 -fi -if [[ ${JOB_ID} == "" ]]; then - echo "No job ID, exiting..." - exit 1 -fi export BUCKET_NAME="cloudexec-${USERNAME}" -echo "Setting up S3 credentials..." -# Spaces uses the AWS S3 API -for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION; do - if [[ -z ${!var} ]]; then - echo "${var} is not set, exiting..." - exit 1 - fi -done - echo "Downloading inputs..." s3cmd get -r "s3://${BUCKET_NAME}/job-${JOB_ID}/input" ~/ @@ -162,6 +205,9 @@ update_state "running" # Create a temporary file to track the completion of the task exit_code_flag="/tmp/cloudexec-exit-code" +######################################## +# Execute Job + # Use Ctrl-C to detach from the tmux session echo "bind-key -n C-c detach" >~/.tmux.conf # Run the tmux command in the background @@ -182,6 +228,9 @@ end_time=$(("${start_time}" + TIMEOUT)) pretty_end_time="$(fmtDate "${end_time}")" echo "Workload is running, timer started at ${pretty_start_time}, we'll time out at ${pretty_end_time}" +######################################## +# Wait for job to finish + # Wait for the temporary file to be created while true; do if [[ -s ${exit_code_flag} ]]; then diff --git a/example/cloudexec.toml b/example/cloudexec.toml index 82e7683..847a7f4 100644 --- a/example/cloudexec.toml +++ b/example/cloudexec.toml @@ -4,32 +4,6 @@ timeout = "48h" [commands] setup = ''' - -######################################## -## Required Configuration and Dependencies - -# set hostname -echo "Setting hostname..." -echo "cloudexec" >/etc/hostname -hostname -F /etc/hostname - -echo "Installing prereqs..." -export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get install -y jq s3cmd tmux python3-pip python3-venv unzip - -echo "Downloading doctl..." -curl -fsSL -o /tmp/doctl-1.92.0-linux-amd64.tar.gz https://github.com/digitalocean/doctl/releases/download/v1.92.0/doctl-1.92.0-linux-amd64.tar.gz -echo "Extracting doctl..." -tar -xzf /tmp/doctl-1.92.0-linux-amd64.tar.gz -C /tmp -echo "Installing doctl..." -mv /tmp/doctl /usr/local/bin -echo "Cleaning up..." -rm /tmp/doctl-1.92.0-linux-amd64.tar.gz - -######################################## -## Common fuzz testing and analysis tools - echo "Installing solc and slither..." python3 -m venv ~/venv source ~/venv/bin/activate From fb64cd18d3c70e536ac203c54efa7f242b233a95 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 22 Sep 2023 15:46:56 -0400 Subject: [PATCH 19/22] don't use non-init cache for init s3 clients --- cmd/cloudexec/init.go | 2 +- cmd/cloudexec/user_data.sh.tmpl | 28 +++++++++++++--------------- pkg/s3/s3.go | 16 ++++++++-------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/cmd/cloudexec/init.go b/cmd/cloudexec/init.go index 3699eba..fbaf5a3 100644 --- a/cmd/cloudexec/init.go +++ b/cmd/cloudexec/init.go @@ -27,7 +27,7 @@ func Init(config config.Config, bucket string) error { fmt.Printf("Creating new %s bucket...\n", bucket) err = s3.CreateBucket(config, bucket) if err != nil { - return fmt.Errorf("Failed to get %s bucket: %w", bucket, err) + return err } } diff --git a/cmd/cloudexec/user_data.sh.tmpl b/cmd/cloudexec/user_data.sh.tmpl index 318b166..3320cb0 100644 --- a/cmd/cloudexec/user_data.sh.tmpl +++ b/cmd/cloudexec/user_data.sh.tmpl @@ -28,23 +28,21 @@ apt-get update apt-get install -y jq s3cmd tmux python3-pip python3-venv unzip # set hostname -if [[ "$(hostname)" != "cloudexec" ]] -then - echo "Setting hostname..." - echo "cloudexec" >/etc/hostname - hostname -F /etc/hostname +if [[ "$(hostname)" != "cloudexec" ]]; then + echo "Setting hostname..." + echo "cloudexec" >/etc/hostname + hostname -F /etc/hostname fi -if ! command -v doctl >/dev/null 2>&1 -then - echo "Downloading doctl..." - curl -fsSL -o /tmp/doctl-1.92.0-linux-amd64.tar.gz https://github.com/digitalocean/doctl/releases/download/v1.92.0/doctl-1.92.0-linux-amd64.tar.gz - echo "Extracting doctl..." - tar -xzf /tmp/doctl-1.92.0-linux-amd64.tar.gz -C /tmp - echo "Installing doctl..." - mv /tmp/doctl /usr/local/bin - echo "Cleaning up..." - rm /tmp/doctl-1.92.0-linux-amd64.tar.gz +if ! command -v doctl >/dev/null 2>&1; then + echo "Downloading doctl..." + curl -fsSL -o /tmp/doctl-1.92.0-linux-amd64.tar.gz https://github.com/digitalocean/doctl/releases/download/v1.92.0/doctl-1.92.0-linux-amd64.tar.gz + echo "Extracting doctl..." + tar -xzf /tmp/doctl-1.92.0-linux-amd64.tar.gz -C /tmp + echo "Installing doctl..." + mv /tmp/doctl /usr/local/bin + echo "Cleaning up..." + rm /tmp/doctl-1.92.0-linux-amd64.tar.gz fi ######################################## diff --git a/pkg/s3/s3.go b/pkg/s3/s3.go index 6b26eb9..43784aa 100644 --- a/pkg/s3/s3.go +++ b/pkg/s3/s3.go @@ -33,7 +33,7 @@ var s3Client *s3.S3 // Note: not safe to use concurrently from multiple goroutines (yet) func initializeS3Client(config config.Config, init bool) (*s3.S3, error) { // Immediately return our cached client if available - if s3Client != nil { + if !init && s3Client != nil { return s3Client, nil } @@ -41,6 +41,7 @@ func initializeS3Client(config config.Config, init bool) (*s3.S3, error) { spacesAccessKey := config.DigitalOcean.SpacesAccessKey spacesSecretKey := config.DigitalOcean.SpacesSecretKey endpointRegion := config.DigitalOcean.SpacesRegion + endpoint := fmt.Sprintf("https://%s.digitaloceanspaces.com", endpointRegion) // Region must be "us-east-1" when creating new Spaces. Otherwise, use the region in your endpoint, such as "nyc3". var spacesRegion string @@ -53,11 +54,13 @@ func initializeS3Client(config config.Config, init bool) (*s3.S3, error) { // Configure the Spaces client spacesConfig := &aws.Config{ Credentials: credentials.NewStaticCredentials(spacesAccessKey, spacesSecretKey, ""), - Endpoint: aws.String(fmt.Sprintf("https://%s.digitaloceanspaces.com", endpointRegion)), + Endpoint: aws.String(endpoint), Region: aws.String(spacesRegion), S3ForcePathStyle: aws.Bool(false), } + fmt.Printf("Creating s3 client with init=%v | endpoint=%s | region=%s\n", init, endpoint, spacesRegion) + // Create a new session and S3 client newSession, err := session.NewSession(spacesConfig) if err != nil { @@ -121,13 +124,10 @@ func CreateBucket(config config.Config, bucket string) error { return err } - // craft bucket creation request - createBucketInput := &s3.CreateBucketInput{ - Bucket: aws.String(bucket), - } - // execution bucket creation - _, err = s3Client.CreateBucket(createBucketInput) + _, err = s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) if err != nil { return fmt.Errorf("Failed to create bucket '%s': %w", bucket, err) } From 929e8e013b5cac98b4b3eb51ff43ad36fbd83471 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 22 Sep 2023 16:36:08 -0400 Subject: [PATCH 20/22] fix s3 client caching --- pkg/s3/s3.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/s3/s3.go b/pkg/s3/s3.go index 43784aa..a6137ee 100644 --- a/pkg/s3/s3.go +++ b/pkg/s3/s3.go @@ -40,8 +40,7 @@ func initializeS3Client(config config.Config, init bool) (*s3.S3, error) { // Unpack required config values spacesAccessKey := config.DigitalOcean.SpacesAccessKey spacesSecretKey := config.DigitalOcean.SpacesSecretKey - endpointRegion := config.DigitalOcean.SpacesRegion - endpoint := fmt.Sprintf("https://%s.digitaloceanspaces.com", endpointRegion) + endpoint := fmt.Sprintf("https://%s.digitaloceanspaces.com", config.DigitalOcean.SpacesRegion) // Region must be "us-east-1" when creating new Spaces. Otherwise, use the region in your endpoint, such as "nyc3". var spacesRegion string @@ -59,19 +58,18 @@ func initializeS3Client(config config.Config, init bool) (*s3.S3, error) { S3ForcePathStyle: aws.Bool(false), } - fmt.Printf("Creating s3 client with init=%v | endpoint=%s | region=%s\n", init, endpoint, spacesRegion) - // Create a new session and S3 client newSession, err := session.NewSession(spacesConfig) if err != nil { return nil, fmt.Errorf("Failed to create S3 client: %w", err) } - // Cache client for subsequent usage iff not the client used for initialization - if !init { - s3Client = s3.New(newSession) + if init { + return s3.New(newSession), nil } + // Cache client for subsequent usage iff not the client used for initialization + s3Client = s3.New(newSession) return s3Client, nil } From 54e126ca3594c485ff8328e5aeed458d8f7f72b8 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 22 Sep 2023 16:37:18 -0400 Subject: [PATCH 21/22] satisfy shellcheck --- cmd/cloudexec/user_data.sh.tmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/cloudexec/user_data.sh.tmpl b/cmd/cloudexec/user_data.sh.tmpl index 3320cb0..53dc4ab 100644 --- a/cmd/cloudexec/user_data.sh.tmpl +++ b/cmd/cloudexec/user_data.sh.tmpl @@ -28,7 +28,8 @@ apt-get update apt-get install -y jq s3cmd tmux python3-pip python3-venv unzip # set hostname -if [[ "$(hostname)" != "cloudexec" ]]; then +current_hostname="$(hostname)" +if [[ ${current_hostname} != "cloudexec" ]]; then echo "Setting hostname..." echo "cloudexec" >/etc/hostname hostname -F /etc/hostname From c51a5d5aa723be1a9faa1bfaff382a2ccdf9afd4 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 22 Sep 2023 17:23:16 -0400 Subject: [PATCH 22/22] wait for unattended-upgr to finish & don't install tools that are already installed --- cmd/cloudexec/user_data.sh.tmpl | 11 +++++- example/cloudexec.toml | 70 ++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/cmd/cloudexec/user_data.sh.tmpl b/cmd/cloudexec/user_data.sh.tmpl index 53dc4ab..c041733 100644 --- a/cmd/cloudexec/user_data.sh.tmpl +++ b/cmd/cloudexec/user_data.sh.tmpl @@ -22,6 +22,12 @@ stderr_log="/tmp/cloudexec-stderr.log" ######################################## # Required setup +# Wait for unattended-upgr to finish install/upgrading stuff in the background +while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + echo "Waiting for unattended-upgr to finish..." + sleep 3 +done + echo "Installing prereqs..." export DEBIAN_FRONTEND=noninteractive apt-get update @@ -66,6 +72,9 @@ for tag in ${TAGS}; do fi done +export BUCKET_NAME="cloudexec-${USERNAME}" +echo "Using bucket ${BUCKET_NAME}" + echo "Setting up DigitalOcean credentials..." # ensure these are set in the environment if [[ -z ${DIGITALOCEAN_ACCESS_TOKEN} ]]; then @@ -186,8 +195,6 @@ echo "Running setup..." mkdir -p ~/output eval "${SETUP_COMMANDS}" -export BUCKET_NAME="cloudexec-${USERNAME}" - echo "Downloading inputs..." s3cmd get -r "s3://${BUCKET_NAME}/job-${JOB_ID}/input" ~/ diff --git a/example/cloudexec.toml b/example/cloudexec.toml index 847a7f4..0781404 100644 --- a/example/cloudexec.toml +++ b/example/cloudexec.toml @@ -4,40 +4,48 @@ timeout = "48h" [commands] setup = ''' -echo "Installing solc and slither..." -python3 -m venv ~/venv -source ~/venv/bin/activate -pip3 install solc-select slither-analyzer crytic-compile -solc-select install 0.8.6 -solc-select use 0.8.6 +if ! command -v slither >/dev/null 2>&1; then + echo "Installing solc and slither..." + python3 -m venv ~/venv + source ~/venv/bin/activate + pip3 install solc-select slither-analyzer crytic-compile + solc-select install 0.8.6 + solc-select use 0.8.6 +fi -echo "Downloading echidna..." -curl -fsSL -o /tmp/echidna.zip https://github.com/crytic/echidna/releases/download/v2.2.1/echidna-2.2.1-Linux.zip -echo "Extracting echidna..." -unzip /tmp/echidna.zip -d /tmp -tar -xzf /tmp/echidna.tar.gz -C /tmp -echo "Installing echidna..." -mv /tmp/echidna /usr/local/bin -rm /tmp/echidna.tar.gz +if ! command -v echidna >/dev/null 2>&1; then + echo "Downloading echidna..." + curl -fsSL -o /tmp/echidna.zip https://github.com/crytic/echidna/releases/download/v2.2.1/echidna-2.2.1-Linux.zip + echo "Extracting echidna..." + unzip /tmp/echidna.zip -d /tmp + tar -xzf /tmp/echidna.tar.gz -C /tmp + echo "Installing echidna..." + mv /tmp/echidna /usr/local/bin + rm /tmp/echidna.tar.gz +fi -echo "Downloading medusa..." -sudo apt-get update; sudo apt-get install -y unzip -curl -fsSL https://github.com/crytic/medusa/releases/download/v0.1.0/medusa-linux-x64.zip -o medusa.zip -unzip medusa.zip -chmod +x medusa -sudo mv medusa /usr/local/bin +if ! command -v medusa >/dev/null 2>&1; then + echo "Downloading medusa..." + sudo apt-get update; sudo apt-get install -y unzip + curl -fsSL https://github.com/crytic/medusa/releases/download/v0.1.0/medusa-linux-x64.zip -o medusa.zip + unzip medusa.zip + chmod +x medusa + sudo mv medusa /usr/local/bin +fi -echo "Installing docker and its dependencies..." -apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common -docker_key="$(curl -fsSL https://download.docker.com/linux/ubuntu/gpg)" -echo "${docker_key}" | apt-key add - -release="$(lsb_release -cs)" -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu ${release} stable" -apt-get update -y -apt-get install -y docker-ce docker-ce-cli containerd.io -user="$(whoami)" -usermod -aG docker "${user}" -systemctl enable docker +if ! command -v docker >/dev/null 2>&1; then + echo "Installing docker and its dependencies..." + apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + docker_key="$(curl -fsSL https://download.docker.com/linux/ubuntu/gpg)" + echo "${docker_key}" | apt-key add - + release="$(lsb_release -cs)" + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu ${release} stable" + apt-get update -y + apt-get install -y docker-ce docker-ce-cli containerd.io + user="$(whoami)" + usermod -aG docker "${user}" + systemctl enable docker + fi ''' # This command is run after the setup script completes.