From 286ba542cc0d42586777f4e4f015d920ac1ad7b1 Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Tue, 20 Feb 2024 11:16:53 -0800 Subject: [PATCH 1/6] Add StemcellAutomation.zip to ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b970abb1..782a25bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ +.DS_Store .idea .envrc stembuild stembuild.exe +StemcellAutomation.zip gordiff gordiff.exe temp From 6befc5edbc3aa6e2a5de971ae3f2fa4c4d4482b1 Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Tue, 20 Feb 2024 11:24:00 -0800 Subject: [PATCH 2/6] Add targets that download StemcellAutomation.zip dependencies from s3 Since tar.exe, bosh-blobstore-s3.exe, bosh-blobstore-dav.exe were (correctly) removed from the BOSH agent repository this had the unfortunate side effect of breaking the stembuild make build target. The latest dependencies to build StemcellAutomation.zip are now downloaded from their respective s3 buckets that the official stembuild CI system uses. --- Makefile | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 75ee7d92..e0f5e048 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,9 @@ STEMCELL_AUTOMATION_PS1 := $(shell ls stemcell-automation/*ps1 | grep -iv Test) BOSH_AGENT_REPO ?= ${HOME}/go/src/github.com/cloudfoundry/bosh-agent LGPO_URL = 'https://download.microsoft.com/download/8/5/C/85C25433-A1B0-4FFA-9429-7E023E7DA8D8/LGPO.zip' BOSH_GCS_URL = 'https://s3.amazonaws.com/bosh-gcscli/bosh-gcscli-0.0.6-windows-amd64.exe' +BOSH_BLOBSTORE_DAV_URL = http://bosh-davcli-artifacts.s3.amazonaws.com +BOSH_BLOBSTORE_S3_URL = http://bosh-s3cli-artifacts.s3.amazonaws.com +BOSH_WINDOWS_DEPENDENCIES_URL = http://bosh-windows-dependencies.s3.amazonaws.com # Ignore things under cis-merge* directory because the paths contain spaces and make doesn't like # that PSMODULES_SOURCES = $(shell find ./modules | grep -v .git | grep -vi "test" | grep -v cis-merge) @@ -83,17 +86,19 @@ assets/local/bosh-agent.exe: $(BOSH_AGENT_SOURCES) cd - mv $(BOSH_AGENT_REPO)/out/bosh-agent assets/local/bosh-agent.exe -assets/local/bosh-blobstore-dav.exe: $(BOSH_AGENT_REPO)/integration/windows/fixtures/bosh-blobstore-dav.exe - @echo "### Creating/Updating assets/local/bosh-blobstore-dav.exe" - $(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/bosh-blobstore-dav.exe assets/local +assets/local/bosh-blobstore-dav.exe: + @echo "### Creating assets/local/bosh-blobstore-dav.exe" + $(eval BOSH_BLOBSTORE_DAV_FILE=$(shell curl -s $(BOSH_BLOBSTORE_DAV_URL) | xq -r '.ListBucketResult.Contents | last | .Key')) + curl -o assets/local/bosh-blobstore-dav.exe -L $(BOSH_BLOBSTORE_DAV_URL)/$(BOSH_BLOBSTORE_DAV_FILE) assets/local/bosh-blobstore-gcs.exe: @echo "### Creating assets/local/bosh-blobstore-gcs.exe" curl -o assets/local/bosh-blobstore-gcs.exe -L $(BOSH_GCS_URL) -assets/local/bosh-blobstore-s3.exe: $(BOSH_AGENT_REPO)/integration/windows/fixtures/bosh-blobstore-s3.exe - @echo "### Creating/Updating assets/local/bosh-blobstore-s3.exe" - $(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/bosh-blobstore-s3.exe assets/local +assets/local/bosh-blobstore-s3.exe: + @echo "### Creating assets/local/bosh-blobstore-s3.exe" + $(eval BOSH_BLOBSTORE_S3_FILE=$(shell curl -s $(BOSH_BLOBSTORE_S3_URL) | xq -r '.ListBucketResult.Contents | last | .Key')) + curl -o assets/local/bosh-blobstore-s3.exe -L $(BOSH_BLOBSTORE_S3_URL)/$(BOSH_BLOBSTORE_S3_FILE) assets/local/bosh-psmodules.zip: $(PSMODULES_SOURCES) @echo "### Creating/Updating assets/local/bosh-psmodules.zip" @@ -118,9 +123,10 @@ assets/local/service_wrapper.xml: $(BOSH_AGENT_REPO)/integration/windows/fixture @echo "### Creating/Updating assets/local/service_wrapper.xml" $(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/service_wrapper.xml assets/local -assets/local/tar.exe: $(BOSH_AGENT_REPO)/integration/windows/fixtures/tar.exe - @echo "### Creating/Updating assets/local/tar.exe" - $(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/tar.exe assets/local +assets/local/tar.exe: + @echo "### Creating assets/local/tar.exe" + $(eval BOSH_WINDOWS_DEPENDENCIES_FILE=$(shell curl -s $(BOSH_WINDOWS_DEPENDENCIES_URL) | xq -r '.ListBucketResult.Contents | map(select(.Key | startswith("tar"))) | last | .Key')) + curl -o assets/local/tar.exe -L $(BOSH_WINDOWS_DEPENDENCIES_URL)/$(BOSH_WINDOWS_DEPENDENCIES_FILE) assets/local/agent.zip: assets/local/bosh-agent.exe assets/local/pipe.exe assets/local/service_wrapper.xml assets/local/service_wrapper.exe assets/local/bosh-blobstore-dav.exe assets/local/bosh-blobstore-gcs.exe assets/local/bosh-blobstore-s3.exe assets/local/job-service-wrapper.exe assets/local/tar.exe @echo "### Creating/Updating assets/local/agent.zip" From 864b58a28ad292196f1af4351cc14749bb77c7cd Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Thu, 22 Feb 2024 14:10:10 -0800 Subject: [PATCH 3/6] Add LGPO.zip to ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 782a25bd..6f219581 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ stembuild stembuild.exe StemcellAutomation.zip +LGPO.zip gordiff gordiff.exe temp From fdc2e2fea7aaf298065e998563cb13c47718fda8 Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Thu, 22 Feb 2024 14:32:04 -0800 Subject: [PATCH 4/6] Add updated local build instructions - Add dependencies required like xq - Update and simplify the example `make build` command --- README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a44a87d8..863c89c6 100644 --- a/README.md +++ b/README.md @@ -225,21 +225,25 @@ dep sync The output should be nothing if there are no out-of-sync dependencies. -## Compile stembuild for macOS +## Compile stembuild locally -You can use stembuild on macOS by following the below steps: +Some of the make targets require the use of curl and [xq](https://github.com/sibprogrammer/xq?tab=readme-ov-file#installation) +to download the latest dependencies from S3 to create the embedded StemcellAutomation.zip resource. The dependencies +downloaded from s3 are the same ones used by [stembuild CI](https://github.com/cloudfoundry/greenhouse-ci/). -- Download or clone the bosh-agent repository +Download or clone the bosh-agent repository ``` git clone https://github.com/cloudfoundry/bosh-agent.git ``` -- Download or clone the stembuild repository and navigate to it + +Download or clone the stembuild repository and navigate to it ``` git clone https://github.com/cloudfoundry/stembuild.git cd stembuild ``` -- Download the latest released artifact from [Stemcell Automation GitHub Repo](https://github.com/cloudfoundry-incubator/bosh-windows-stemcell-automation/releases) -- Use `make build` to build stembuild for macOS, providing the corresponding values for the bosh-agent path, Stemcell Version you would like to build, and the Stemcell Automation package you downloaded + +Use `make build` to build stembuild, providing the corresponding values for the bosh-agent path and stemcell version you +would like to build, for example: ``` -BOSH_AGENT_REPO=PathToBoshAgentRepo make STEMCELL_VERSION=StemcellVersion AUTOMATION_PATH=PathToStemcellAutomationZip build +BOSH_AGENT_REPO=../bosh-agent STEMCELL_VERSION=2019.70 make build ``` From 16269cbc4d1cfeff1e18732f1e9143a164caeb23 Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Tue, 12 Mar 2024 13:13:29 -0700 Subject: [PATCH 5/6] Add corrected xq pathing selection - Fix sorting and tool selection to use latest versions --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e0f5e048..f88ed757 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ assets/local/bosh-agent.exe: $(BOSH_AGENT_SOURCES) assets/local/bosh-blobstore-dav.exe: @echo "### Creating assets/local/bosh-blobstore-dav.exe" - $(eval BOSH_BLOBSTORE_DAV_FILE=$(shell curl -s $(BOSH_BLOBSTORE_DAV_URL) | xq -r '.ListBucketResult.Contents | last | .Key')) + $(eval BOSH_BLOBSTORE_DAV_FILE=$(shell curl -s $(BOSH_BLOBSTORE_DAV_URL) | xq --xpath '//Key' | sort --version-sort | tail -1)) curl -o assets/local/bosh-blobstore-dav.exe -L $(BOSH_BLOBSTORE_DAV_URL)/$(BOSH_BLOBSTORE_DAV_FILE) assets/local/bosh-blobstore-gcs.exe: @@ -97,7 +97,7 @@ assets/local/bosh-blobstore-gcs.exe: assets/local/bosh-blobstore-s3.exe: @echo "### Creating assets/local/bosh-blobstore-s3.exe" - $(eval BOSH_BLOBSTORE_S3_FILE=$(shell curl -s $(BOSH_BLOBSTORE_S3_URL) | xq -r '.ListBucketResult.Contents | last | .Key')) + $(eval BOSH_BLOBSTORE_S3_FILE=$(shell curl -s $(BOSH_BLOBSTORE_S3_URL) | xq --xpath '//Key' | sort --version-sort | tail -1)) curl -o assets/local/bosh-blobstore-s3.exe -L $(BOSH_BLOBSTORE_S3_URL)/$(BOSH_BLOBSTORE_S3_FILE) assets/local/bosh-psmodules.zip: $(PSMODULES_SOURCES) @@ -125,7 +125,7 @@ assets/local/service_wrapper.xml: $(BOSH_AGENT_REPO)/integration/windows/fixture assets/local/tar.exe: @echo "### Creating assets/local/tar.exe" - $(eval BOSH_WINDOWS_DEPENDENCIES_FILE=$(shell curl -s $(BOSH_WINDOWS_DEPENDENCIES_URL) | xq -r '.ListBucketResult.Contents | map(select(.Key | startswith("tar"))) | last | .Key')) + $(eval BOSH_WINDOWS_DEPENDENCIES_FILE=$(shell curl -s $(BOSH_WINDOWS_DEPENDENCIES_URL) | xq --xpath '//Key[contains(text(), "tar")]' | sort --version-sort | tail -1)) curl -o assets/local/tar.exe -L $(BOSH_WINDOWS_DEPENDENCIES_URL)/$(BOSH_WINDOWS_DEPENDENCIES_FILE) assets/local/agent.zip: assets/local/bosh-agent.exe assets/local/pipe.exe assets/local/service_wrapper.xml assets/local/service_wrapper.exe assets/local/bosh-blobstore-dav.exe assets/local/bosh-blobstore-gcs.exe assets/local/bosh-blobstore-s3.exe assets/local/job-service-wrapper.exe assets/local/tar.exe From b60b6ca30a0883572b7f65119111bbd95c4ac146 Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Tue, 12 Mar 2024 13:54:35 -0700 Subject: [PATCH 6/6] Add STEMBUILD_VERSION env var to make usage --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 863c89c6..e052bd5a 100644 --- a/README.md +++ b/README.md @@ -245,5 +245,5 @@ cd stembuild Use `make build` to build stembuild, providing the corresponding values for the bosh-agent path and stemcell version you would like to build, for example: ``` -BOSH_AGENT_REPO=../bosh-agent STEMCELL_VERSION=2019.70 make build +BOSH_AGENT_REPO=../bosh-agent STEMBUILD_VERSION=2019.70 make build ```