From 6e96e6347e850b9f1c0397f100b42b93abef03d2 Mon Sep 17 00:00:00 2001 From: bohendo Date: Thu, 28 Mar 2024 16:27:17 -0400 Subject: [PATCH 1/5] add helper fn for getting latest release artifact --- packer/provision.sh | 44 ++++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/packer/provision.sh b/packer/provision.sh index 0880f2c..f36ac06 100644 --- a/packer/provision.sh +++ b/packer/provision.sh @@ -2,6 +2,29 @@ # shellcheck source=/dev/null set -e +######################################## +## Helper Functions + +function get_latest_version { + project="$1" + github_api="https://api.github.com/repos/${project}/releases/latest" + curl -s "$github_api" | jq '.tag_name' | tr -d 'v' | tr -d '"' +} + +function get_latest_artifact { + project="$1" + github_api="https://api.github.com/repos/${project}/releases/latest" + curl -s "$github_api" | jq '.assets[] | select(.name | test("linux")) | .browser_download_url' | grep -v "sigstore" | tr -d '"' +} + +######################################## +## Versions + +doctl_version="1.92.0" +solc_version="0.8.6" +echidna_version="$(get_latest_version "crytic/echidna")" +medusa_version="$(get_latest_version "crytic/medusa")" + ######################################## ## Required Configuration and Dependencies @@ -27,28 +50,25 @@ rm /tmp/doctl-1.92.0-linux-amd64.tar.gz ######################################## ## Common fuzz testing and analysis tools -echo "Installing solc and slither..." +echo "Installing slither and solc v${solc_version}..." python3 -m venv ~/venv source ~/venv/bin/activate pip3 install solc-select slither-analyzer crytic-compile solc-select use latest --always-install -echo "Downloading echidna..." -curl -fsSL https://github.com/crytic/echidna/releases/download/v2.2.3/echidna-2.2.3-x86_64-linux.tar.gz -o /tmp/echidna.tar.gz -echo "Extracting echidna..." +echo "Installing echidna v${echidna_version}..." +curl -fsSL -o /tmp/echidna.tar.gz "$(get_latest_artifact "crytic/echidna")" tar -xzf /tmp/echidna.tar.gz -C /tmp -echo "Installing echidna..." mv /tmp/echidna /usr/local/bin +chmod +x /usr/local/bin/echidna rm /tmp/echidna.tar.gz -echo "Downloading medusa..." -curl -fsSL https://github.com/crytic/medusa/releases/download/v0.1.3/medusa-linux-x64.tar.gz -o /tmp/medusa.tar.gz -echo "Extracting medusa..." -tar -xzf /tmp/medusa.tar.gz -C /tmp -echo "Installing medusa..." -chmod +x /tmp/medusa +echo "Installing medusa v${medusa_version}..." +curl -fsSL -o /tmp/medusa.zip "$(get_latest_artifact "crytic/medusa")" +unzip /tmp/medusa.zip -d /tmp sudo mv /tmp/medusa /usr/local/bin -rm /tmp/medusa.tar.gz +chmod +x /usr/local/bin/medusa +rm /tmp/medusa.zip echo "Installing docker and its dependencies..." apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common From ed71922c973e93cac2f0739abafd6f0bdc594a64 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 29 Mar 2024 10:22:57 -0400 Subject: [PATCH 2/5] upgrade devTools eg packer --- flake.lock | 8 ++++---- flake.nix | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 3bf0fc5..192128a 100644 --- a/flake.lock +++ b/flake.lock @@ -2,17 +2,17 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1685566663, - "narHash": "sha256-btHN1czJ6rzteeCuE/PNrdssqYD2nIA4w48miQAFloM=", + "lastModified": 1707530871, + "narHash": "sha256-W9pVy2PbepxBjoI/AQyQJYwyp7e1JdLYRHtUjUc4yso=", "owner": "nixos", "repo": "nixpkgs", - "rev": "4ecab3273592f27479a583fb6d975d4aba3486fe", + "rev": "11cd405226b6663b1ba2073dc7d8b0d7a78175d9", "type": "github" }, "original": { "owner": "nixos", "repo": "nixpkgs", - "rev": "4ecab3273592f27479a583fb6d975d4aba3486fe", + "rev": "11cd405226b6663b1ba2073dc7d8b0d7a78175d9", "type": "github" } }, diff --git a/flake.nix b/flake.nix index acda03e..9336b1a 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "CloudExec VPS provisioning helper"; inputs = { - nixpkgs.url = "github:nixos/nixpkgs/4ecab3273592f27479a583fb6d975d4aba3486fe"; # v23.05 + nixpkgs.url = "github:nixos/nixpkgs/11cd405226b6663b1ba2073dc7d8b0d7a78175d9"; # 240209 utils.url = "github:numtide/flake-utils/04c1b180862888302ddfb2e3ad9eaa63afc60cf8"; # v1.0.0 }; @@ -134,7 +134,6 @@ go-tools gopls go-outline - gocode gopkgs gocode-gomod godef From dda1487566736f630243924c42e9140eca2ef7d1 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 29 Mar 2024 11:01:16 -0400 Subject: [PATCH 3/5] [packer] dynamically fetch latest release artifacts from github --- packer/provision.sh | 52 ++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/packer/provision.sh b/packer/provision.sh index f36ac06..67dae24 100644 --- a/packer/provision.sh +++ b/packer/provision.sh @@ -5,25 +5,19 @@ set -e ######################################## ## Helper Functions -function get_latest_version { +function github_api { project="$1" github_api="https://api.github.com/repos/${project}/releases/latest" - curl -s "$github_api" | jq '.tag_name' | tr -d 'v' | tr -d '"' + curl -s "$github_api" } -function get_latest_artifact { - project="$1" - github_api="https://api.github.com/repos/${project}/releases/latest" - curl -s "$github_api" | jq '.assets[] | select(.name | test("linux")) | .browser_download_url' | grep -v "sigstore" | tr -d '"' +function get_latest_version { + github_api "$1" | jq '.tag_name' | tr -d 'v' | tr -d '"' } -######################################## -## Versions - -doctl_version="1.92.0" -solc_version="0.8.6" -echidna_version="$(get_latest_version "crytic/echidna")" -medusa_version="$(get_latest_version "crytic/medusa")" +function get_latest_artifact { + github_api "$1" | jq '.assets[] | select(.name | test("linux")) | .browser_download_url' | tr -d '"' +} ######################################## ## Required Configuration and Dependencies @@ -38,37 +32,37 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y jq s3cmd tmux python3-pip python3-venv -echo "Downloading doctl..." -curl -fsSL -o /tmp/doctl-1.92.0-linux-amd64.tar.gz https://github.com/digitalocean/doctl/releases/download/v1.92.0/doctl-1.92.0-linux-amd64.tar.gz -echo "Extracting doctl..." -tar -xzf /tmp/doctl-1.92.0-linux-amd64.tar.gz -C /tmp -echo "Installing doctl..." +echo "Installing doctl v$(get_latest_version "digitalocean/doctl")..." +curl -fsSL -o /tmp/doctl.tar.gz "$(get_latest_artifact "digitalocean/doctl" | grep "amd64")" +tar -xzf /tmp/doctl.tar.gz -C /tmp mv /tmp/doctl /usr/local/bin -echo "Cleaning up..." -rm /tmp/doctl-1.92.0-linux-amd64.tar.gz +rm /tmp/doctl.tar.gz +echo "Done installing: $(doctl version)" ######################################## ## Common fuzz testing and analysis tools -echo "Installing slither and solc v${solc_version}..." +echo "Installing slither..." python3 -m venv ~/venv source ~/venv/bin/activate pip3 install solc-select slither-analyzer crytic-compile solc-select use latest --always-install -echo "Installing echidna v${echidna_version}..." -curl -fsSL -o /tmp/echidna.tar.gz "$(get_latest_artifact "crytic/echidna")" +echo "Installing echidna v$(get_latest_version "crytic/echidna")..." +curl -fsSL -o /tmp/echidna.tar.gz "$(get_latest_artifact "crytic/echidna" | grep -v "sigstore")" tar -xzf /tmp/echidna.tar.gz -C /tmp mv /tmp/echidna /usr/local/bin chmod +x /usr/local/bin/echidna rm /tmp/echidna.tar.gz +echo "Done installing: $(echidna --version)" -echo "Installing medusa v${medusa_version}..." -curl -fsSL -o /tmp/medusa.zip "$(get_latest_artifact "crytic/medusa")" -unzip /tmp/medusa.zip -d /tmp +echo "Installing medusa v$(get_latest_version "crytic/medusa")..." +curl -fsSL -o /tmp/medusa.tar.gz "$(get_latest_artifact "crytic/medusa" | grep -v "sigstore")" +tar -xzf /tmp/medusa.tar.gz -C /tmp sudo mv /tmp/medusa /usr/local/bin chmod +x /usr/local/bin/medusa -rm /tmp/medusa.zip +rm /tmp/medusa.tar.gz +echo "Done installing: $(medusa --version)" echo "Installing docker and its dependencies..." apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common @@ -81,3 +75,7 @@ apt-get install -y docker-ce docker-ce-cli containerd.io user="$(whoami)" usermod -aG docker "${user}" systemctl enable docker +echo "Done installing docker:" +docker version + +echo "Done provisioning" From 7a97b0c0304775660567e5921c74fbd04d4938f6 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 29 Mar 2024 11:32:25 -0400 Subject: [PATCH 4/5] add just command for running packer --- flake.nix | 2 +- justfile | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 9336b1a..a8bc6a7 100644 --- a/flake.nix +++ b/flake.nix @@ -101,7 +101,7 @@ url = "git+ssh://git@github.com/trailofbits/medusa"; rev = "72e9b8586ad93b37ff9063ccf3f5b471f934c264"; }; - vendorSha256 = "sha256-IKB8c6oxF5h88FdzUAmNA96BpNo/LIbwzuDCMFsdZNE="; + vendorHash = "sha256-IKB8c6oxF5h88FdzUAmNA96BpNo/LIbwzuDCMFsdZNE="; nativeBuildInputs = [ packages.crytic-compile pkgs.solc diff --git a/justfile b/justfile index 3b055af..67d2e96 100644 --- a/justfile +++ b/justfile @@ -9,11 +9,13 @@ trunk: trunk fmt trunk check +pack opSecretReference="op://Private/DigitalOcean/ApiKey": + cd packer && packer build -var do_api_token=$(op read {{opSecretReference}}) cloudexec.pkr.hcl + build: nix build -install: - nix build +install: build echo nix profile remove $(nix profile list | grep cloudexec | cut -d " " -f 1) nix profile remove $(nix profile list | grep cloudexec | cut -d " " -f 1) echo nix profile install ./result From 35e86164923982785a07df318907b06799256955 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 29 Mar 2024 11:56:07 -0400 Subject: [PATCH 5/5] trunk fmt --- .shellcheckrc | 3 ++- cmd/cloudexec/cancel.go | 2 +- cmd/cloudexec/main.go | 2 +- packer/provision.sh | 40 +++++++++++++++++++++++++++++----------- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.shellcheckrc b/.shellcheckrc index e68fb49..288c3f5 100644 --- a/.shellcheckrc +++ b/.shellcheckrc @@ -1,2 +1,3 @@ enable=all -disable=SC2154 # Disabled because it's disabled by trunk +disable=SC2154 # Disabled because it's disabled in trunk by default +disable=SC2312 # Disabled because we often don't need return values diff --git a/cmd/cloudexec/cancel.go b/cmd/cloudexec/cancel.go index b095219..d256291 100644 --- a/cmd/cloudexec/cancel.go +++ b/cmd/cloudexec/cancel.go @@ -13,7 +13,7 @@ import ( func CancelJob(config config.Config, existingState *state.State, job *state.Job, force bool) error { if job.Status != state.Provisioning && job.Status != state.Running { log.Info("Job %v is not running, it is %s", job.ID, job.Status) - return nil + return nil } log.Warn("Destroying droplet %s associated with job %v: IP=%v | CreatedAt=%s", job.Droplet.Name, job.ID, job.Droplet.IP, job.Droplet.Created) if !force { // Ask for confirmation before cleaning this job if no force flag diff --git a/cmd/cloudexec/main.go b/cmd/cloudexec/main.go index 5947c13..6f628d4 100644 --- a/cmd/cloudexec/main.go +++ b/cmd/cloudexec/main.go @@ -129,7 +129,7 @@ func main() { return err } err = Launch(config, dropletSize, dropletRegion, lc) - return err + return err }, }, diff --git a/packer/provision.sh b/packer/provision.sh index 67dae24..c53dbaa 100644 --- a/packer/provision.sh +++ b/packer/provision.sh @@ -6,17 +6,17 @@ set -e ## Helper Functions function github_api { - project="$1" - github_api="https://api.github.com/repos/${project}/releases/latest" - curl -s "$github_api" + project="$1" + github_api="https://api.github.com/repos/${project}/releases/latest" + curl -s "${github_api}" } function get_latest_version { - github_api "$1" | jq '.tag_name' | tr -d 'v' | tr -d '"' + github_api "$1" | jq '.tag_name' | tr -d 'v' | tr -d '"' } function get_latest_artifact { - github_api "$1" | jq '.assets[] | select(.name | test("linux")) | .browser_download_url' | tr -d '"' + github_api "$1" | jq '.assets[] | select(.name | test("linux")) | .browser_download_url' | tr -d '"' } ######################################## @@ -32,8 +32,14 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y jq s3cmd tmux python3-pip python3-venv -echo "Installing doctl v$(get_latest_version "digitalocean/doctl")..." -curl -fsSL -o /tmp/doctl.tar.gz "$(get_latest_artifact "digitalocean/doctl" | grep "amd64")" +echo "Installing doctl v$( + set -e + get_latest_version "digitalocean/doctl" +)..." +curl -fsSL -o /tmp/doctl.tar.gz "$( + set -e + get_latest_artifact "digitalocean/doctl" | grep "amd64" +)" tar -xzf /tmp/doctl.tar.gz -C /tmp mv /tmp/doctl /usr/local/bin rm /tmp/doctl.tar.gz @@ -48,16 +54,28 @@ source ~/venv/bin/activate pip3 install solc-select slither-analyzer crytic-compile solc-select use latest --always-install -echo "Installing echidna v$(get_latest_version "crytic/echidna")..." -curl -fsSL -o /tmp/echidna.tar.gz "$(get_latest_artifact "crytic/echidna" | grep -v "sigstore")" +echo "Installing echidna v$( + set -e + get_latest_version "crytic/echidna" +)..." +curl -fsSL -o /tmp/echidna.tar.gz "$( + set -e + get_latest_artifact "crytic/echidna" | grep -v "sigstore" +)" tar -xzf /tmp/echidna.tar.gz -C /tmp mv /tmp/echidna /usr/local/bin chmod +x /usr/local/bin/echidna rm /tmp/echidna.tar.gz echo "Done installing: $(echidna --version)" -echo "Installing medusa v$(get_latest_version "crytic/medusa")..." -curl -fsSL -o /tmp/medusa.tar.gz "$(get_latest_artifact "crytic/medusa" | grep -v "sigstore")" +echo "Installing medusa v$( + set -e + get_latest_version "crytic/medusa" +)..." +curl -fsSL -o /tmp/medusa.tar.gz "$( + set -e + get_latest_artifact "crytic/medusa" | grep -v "sigstore" +)" tar -xzf /tmp/medusa.tar.gz -C /tmp sudo mv /tmp/medusa /usr/local/bin chmod +x /usr/local/bin/medusa