Skip to content

Commit 1b49654

Browse files
ystrosystros
committed
Update construction of StemcellAutomation.zip
This updates the construction of StemcellAutomation.zip to require user-provided files. This is closer to how the official binaries for `stembuild` are built, and should be safer than the previous way assets were being downloaded. A script has been added to build the zip file when provided with the appropriate assets.
1 parent 2a075fa commit 1b49654

File tree

4 files changed

+92
-158
lines changed

4 files changed

+92
-158
lines changed

‎Makefile

+6-116
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,6 @@ GOSRC = $(shell find . -name "*.go" ! -name "*test.go" ! -name "*fake*" ! -path
22
STEMCELL_VERSION = $(shell echo "$${STEMBUILD_VERSION}")
33
LD_FLAGS = "-w -s -X github.com/cloudfoundry/stembuild/version.Version=${STEMCELL_VERSION}"
44

5-
# These are the sources for StemcellAutomation.zip
6-
STEMCELL_AUTOMATION_PS1 := $(shell ls stemcell-automation/*ps1 | grep -iv Test)
7-
BOSH_AGENT_REPO ?= ${HOME}/go/src/github.com/cloudfoundry/bosh-agent
8-
LGPO_URL = 'https://download.microsoft.com/download/8/5/C/85C25433-A1B0-4FFA-9429-7E023E7DA8D8/LGPO.zip'
9-
BOSH_GCS_URL = 'https://s3.amazonaws.com/bosh-gcscli/bosh-gcscli-0.0.6-windows-amd64.exe'
10-
BOSH_BLOBSTORE_DAV_URL = http://bosh-davcli-artifacts.s3.amazonaws.com
11-
BOSH_BLOBSTORE_S3_URL = http://bosh-s3cli-artifacts.s3.amazonaws.com
12-
BOSH_WINDOWS_DEPENDENCIES_URL = http://bosh-windows-dependencies.s3.amazonaws.com
13-
# Ignore things under cis-merge* directory because the paths contain spaces and make doesn't like
14-
# that
15-
PSMODULES_SOURCES = $(shell find ./modules | grep -v .git | grep -vi "test" | grep -v cis-merge)
16-
BOSH_AGENT_SOURCES = $(shell find $(BOSH_AGENT_REPO) | egrep -v ".git|test.go|fake|.md")
17-
185
ifeq ($(OS),Windows_NT)
196
COMMAND = out/stembuild.exe
207
CP = cp
@@ -54,12 +41,10 @@ integration-badger : generate-fake-stemcell-automation
5441
generate-fake-stemcell-automation:
5542
$(CP) integration/construct/assets/StemcellAutomation.zip assets/
5643

57-
generate: assets/StemcellAutomation.zip
58-
59-
out/stembuild : generate $(GOSRC)
44+
out/stembuild : assets/StemcellAutomation.zip $(GOSRC)
6045
CGO_ENABLED=0 go build -o $(COMMAND) -ldflags $(LD_FLAGS) .
6146

62-
out/stembuild.exe : generate $(GOSRC)
47+
out/stembuild.exe : assets/StemcellAutomation.zip $(GOSRC)
6348
GOOS=windows CGO_ENABLED=0 go build -o out/stembuild.exe -ldflags $(LD_FLAGS) .
6449

6550
test : units
@@ -73,104 +58,9 @@ units : format generate-fake-stemcell-automation
7358
contract :
7459
go run github.com/onsi/ginkgo/v2/ginkgo -r --randomize-all --randomize-suites --keep-going --flake-attempts 2 iaas_cli
7560

76-
.PHONY : all build build-integration clean format generate generate-fake-stemcell-automation
61+
.PHONY : all build build-integration clean format generate-fake-stemcell-automation
7762
.PHONY : test units units-full integration integration-tests-full
7863

79-
# ===============================================================================
80-
# The following to create the StemcellAutomation.zip that's packaged in stembuild
81-
# ===============================================================================
82-
83-
assets/local/bosh-agent.exe: $(BOSH_AGENT_SOURCES)
84-
cd $(BOSH_AGENT_REPO) && \
85-
GOOS=windows GOARCH=amd64 bin/build && \
86-
cd -
87-
mv $(BOSH_AGENT_REPO)/out/bosh-agent assets/local/bosh-agent.exe
88-
89-
assets/local/bosh-blobstore-dav.exe:
90-
@echo "### Creating assets/local/bosh-blobstore-dav.exe"
91-
$(eval BOSH_BLOBSTORE_DAV_FILE=$(shell curl -s $(BOSH_BLOBSTORE_DAV_URL) | xq --xpath '//Key' | sort --version-sort | tail -1))
92-
curl -o assets/local/bosh-blobstore-dav.exe -L $(BOSH_BLOBSTORE_DAV_URL)/$(BOSH_BLOBSTORE_DAV_FILE)
93-
94-
assets/local/bosh-blobstore-gcs.exe:
95-
@echo "### Creating assets/local/bosh-blobstore-gcs.exe"
96-
curl -o assets/local/bosh-blobstore-gcs.exe -L $(BOSH_GCS_URL)
97-
98-
assets/local/bosh-blobstore-s3.exe:
99-
@echo "### Creating assets/local/bosh-blobstore-s3.exe"
100-
$(eval BOSH_BLOBSTORE_S3_FILE=$(shell curl -s $(BOSH_BLOBSTORE_S3_URL) | xq --xpath '//Key' | sort --version-sort | tail -1))
101-
curl -o assets/local/bosh-blobstore-s3.exe -L $(BOSH_BLOBSTORE_S3_URL)/$(BOSH_BLOBSTORE_S3_FILE)
102-
103-
assets/local/bosh-psmodules.zip: $(PSMODULES_SOURCES)
104-
@echo "### Creating/Updating assets/local/bosh-psmodules.zip"
105-
cd modules && zip -r ../bosh-psmodules.zip . && cd ..
106-
mv bosh-psmodules.zip assets/local/bosh-psmodules.zip
107-
108-
assets/local/job-service-wrapper.exe: $(BOSH_AGENT_REPO)/integration/windows/fixtures/job-service-wrapper.exe
109-
@echo "### Creating/Updating assets/local/job-service-wrapper.exe"
110-
$(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/job-service-wrapper.exe assets/local
111-
112-
assets/local/pipe.exe: $(BOSH_AGENT_SOURCES)
113-
cd $(BOSH_AGENT_REPO) && \
114-
GOOS=windows GOARCH=amd64 bin/build && \
115-
cd -
116-
mv $(BOSH_AGENT_REPO)/out/bosh-agent-pipe assets/local/pipe.exe
117-
118-
assets/local/service_wrapper.exe: $(BOSH_AGENT_REPO)/integration/windows/fixtures/service_wrapper.exe
119-
@echo "### Creating/Updating assets/local/service_wrapper.exe"
120-
$(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/service_wrapper.exe assets/local
121-
122-
assets/local/service_wrapper.xml: $(BOSH_AGENT_REPO)/integration/windows/fixtures/service_wrapper.xml
123-
@echo "### Creating/Updating assets/local/service_wrapper.xml"
124-
$(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/service_wrapper.xml assets/local
125-
126-
assets/local/tar.exe:
127-
@echo "### Creating assets/local/tar.exe"
128-
$(eval BOSH_WINDOWS_DEPENDENCIES_FILE=$(shell curl -s $(BOSH_WINDOWS_DEPENDENCIES_URL) | xq --xpath '//Key[contains(text(), "tar")]' | sort --version-sort | tail -1))
129-
curl -o assets/local/tar.exe -L $(BOSH_WINDOWS_DEPENDENCIES_URL)/$(BOSH_WINDOWS_DEPENDENCIES_FILE)
130-
131-
assets/local/agent.zip: assets/local/bosh-agent.exe assets/local/pipe.exe assets/local/service_wrapper.xml assets/local/service_wrapper.exe assets/local/bosh-blobstore-dav.exe assets/local/bosh-blobstore-gcs.exe assets/local/bosh-blobstore-s3.exe assets/local/job-service-wrapper.exe assets/local/tar.exe
132-
@echo "### Creating/Updating assets/local/agent.zip"
133-
mkdir -p assets/temp/deps
134-
$(CP) assets/local/service_wrapper.exe \
135-
assets/local/service_wrapper.xml \
136-
assets/local/bosh-agent.exe \
137-
assets/temp
138-
$(CP) assets/local/bosh-blobstore-dav.exe \
139-
assets/local/bosh-blobstore-gcs.exe \
140-
assets/local/bosh-blobstore-s3.exe \
141-
assets/local/job-service-wrapper.exe \
142-
assets/local/pipe.exe \
143-
assets/local/tar.exe \
144-
assets/temp/deps
145-
cd assets/temp && zip -r ../local/agent.zip * && cd -
146-
rm -rf assets/temp
147-
148-
assets/local/LGPO.zip:
149-
@echo "### Creating assets/local/LGPO.zip"
150-
curl -o assets/local/LGPO.zip -L $(LGPO_URL)
151-
152-
assets/local/OpenSSH-Win64.zip: $(BOSH_AGENT_REPO)/integration/windows/fixtures/OpenSSH-Win64.zip
153-
@echo "### Creating/Updating assets/local/OpenSSH-Win64.zip"
154-
$(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/OpenSSH-Win64.zip assets/local
155-
156-
assets/local/deps.json: assets/local/agent.zip assets/local/bosh-psmodules.zip assets/local/LGPO.zip assets/local/OpenSSH-Win64.zip
157-
@echo "### Creating/Updating assets/local/deps.json"
158-
@#Note: The order of the following matters, change the script before changing these
159-
stemcell-automation/generate-dep-json.bash \
160-
assets/local/OpenSSH-Win64.zip \
161-
assets/local/bosh-psmodules.zip \
162-
assets/local/agent.zip \
163-
assets/local/LGPO.zip \
164-
> assets/local/deps.json
165-
166-
assets/StemcellAutomation.zip: $(STEMCELL_AUTOMATION_PS1) assets/local/OpenSSH-Win64.zip assets/local/bosh-psmodules.zip assets/local/deps.json assets/local/agent.zip
167-
@echo "### Creating/Updating assets/StemcellAutomation.zip"
168-
mkdir -p assets/temp
169-
cp -a $(STEMCELL_AUTOMATION_PS1) \
170-
assets/local/OpenSSH-Win64.zip \
171-
assets/local/bosh-psmodules.zip \
172-
assets/local/deps.json \
173-
assets/local/agent.zip \
174-
assets/temp
175-
cd assets/temp && zip ../StemcellAutomation.zip * && cd -
176-
rm -rf assets/temp
64+
assets/StemcellAutomation.zip:
65+
@echo "Unable to locate assets/StemcellAutomation.zip. This file is bundled into the final stembuild executable and must be present. Please see README.md on how to construct this file."
66+
exit 1

‎README.md

+62-17
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ stembuild construct -vm-ip <IP of VM> -vm-username <vm username> -vm-password <v
3737
```
3838

3939
### Requirements
40-
- LGPO.zip in current working directory
40+
- LGPO.zip in current working directory. This is a zip of LGPO.exe from Microsoft that is compatible with the current version of Windows. https://www.microsoft.com/en-us/download/details.aspx?id=55319
41+
- Constructed assets/StemcellAutomation.zip (contents described below)
4142
- Running Windows VM with:
4243
- Up-to-date Operating System
4344
- Reachable by IP over port 5985
@@ -113,14 +114,14 @@ Flags:
113114
114115
```
115116

116-
### Compiling & Running Stembuild Locally
117+
### Running Stembuild Locally
117118

118119
Assuming you've followed [these instructions](https://bosh.io/docs/windows-stemcell-create/) and you've created a Windows VM at 10.9.9.115 whose Administrator's password is "c1oudc0w".
119120

120121
```bash
121-
export TARGET_VM_PASSWORD=c1oudc0w VCENTER_PASSWORD='Admin!23'
122-
BOSH_AGENT_REPO=~/workspace/bosh-agent STEMBUILD_VERSION=2019.2 make
123-
GOVC_INSECURE=true out/stembuild -debug \
122+
export TARGET_VM_PASSWORD=c1oudc0w VCENTER_PASSWORD='Admin!23'
123+
124+
GOVC_INSECURE=true stembuild -debug \
124125
construct \
125126
-vm-ip 10.9.9.115 \
126127
-vm-username Administrator \
@@ -129,7 +130,8 @@ GOVC_INSECURE=true out/stembuild -debug \
129130
-vcenter-username [email protected] \
130131
-vcenter-password $VCENTER_PASSWORD \
131132
-vm-inventory-path "/dc/vm/Discovered virtual machine/w2019-stemcell"
132-
GOVC_INSECURE=true out/stembuild -debug \
133+
134+
GOVC_INSECURE=true stembuild -debug \
133135
package \
134136
-vcenter-url vcenter-70.nono.io \
135137
-vcenter-username [email protected] \
@@ -227,23 +229,66 @@ The output should be nothing if there are no out-of-sync dependencies.
227229

228230
## Compile stembuild locally
229231

230-
Some of the make targets require the use of curl and [xq](https://github.com/sibprogrammer/xq?tab=readme-ov-file#installation)
231-
to download the latest dependencies from S3 to create the embedded StemcellAutomation.zip resource. The dependencies
232-
downloaded from s3 are the same ones used by [stembuild CI](https://github.com/cloudfoundry/greenhouse-ci/).
233-
234-
Download or clone the bosh-agent repository
235-
```
236-
git clone https://github.com/cloudfoundry/bosh-agent.git
237-
```
238-
239232
Download or clone the stembuild repository and navigate to it
240233
```
241234
git clone https://github.com/cloudfoundry/stembuild.git
242235
cd stembuild
243236
```
244237

245-
Use `make build` to build stembuild, providing the corresponding values for the bosh-agent path and stemcell version you
238+
You will need to construct `assets/StemcellAutomation.zip`. This file represents various BOSH executables necessary to build a working stemcell.
239+
240+
**assets/StemcellAutomation.zip files:**
241+
| File | Source / Description |
242+
|-|-|
243+
| OpenSSH-Win64.zip | https://github.com/PowerShell/Win32-OpenSSH/releases |
244+
| bosh-psmodules.zip | https://github.com/cloudfoundry/bosh-psmodules/tree/master/modules |
245+
| agent.zip | A zip constructed using various BOSH executables. See list of necessary files below. |
246+
| deps.json | A JSON file with the SHA256 checksums and optionally the version for each component in this zip. See format below. |
247+
248+
**agent.zip files:**
249+
| File | Source / Description |
250+
|-|-|
251+
| bosh-agent.exe | https://github.com/cloudfoundry/bosh-agent/ |
252+
| sha | Git commit SHA from the bosh-agent repo |
253+
| deps/pipe.exe | https://github.com/cloudfoundry/bosh-agent/tree/main/jobsupervisor/pipe |
254+
| deps/bosh-blobstore-gcs.exe | https://github.com/cloudfoundry/bosh-gcscli |
255+
| deps/bosh-blobstore-dav.exe | https://github.com/cloudfoundry/bosh-davcli |
256+
| deps/bosh-blobstore-s3.exe | https://github.com/cloudfoundry/bosh-s3cli |
257+
| deps/tar.exe | https://github.com/cloudfoundry/bsdtar/ |
258+
| deps/job-service-wrapper.exe | https://github.com/bosh-dep-forks/winsw |
259+
| service_wrapper.exe | https://github.com/bosh-dep-forks/winsw |
260+
| service_wrapper.xml | https://github.com/cloudfoundry/bosh-agent/blob/main/integration/windows/fixtures/service_wrapper.xml |
261+
262+
**deps.json format:**
263+
```json
264+
{
265+
"OpenSSH-Win64.zip": {
266+
"sha": "SOME-SHA256",
267+
},
268+
"bosh-psmodules.zip": {
269+
"sha": "SOME-SHA256",
270+
},
271+
"agent.zip": {
272+
"sha": "SOME-SHA256",
273+
},
274+
"LGPO.zip": {
275+
"sha": "SOME-SHA256",
276+
"version": "3"
277+
}
278+
}
279+
```
280+
281+
Once you have these files, run `./bin/build-stemcell-automation-zip.sh`:
282+
```bash
283+
OPENSSH_ZIP=OpenSSH-Win64.zip \
284+
BOSH_PSMODULES_ZIP="bosh-psmodules.zip" \
285+
AGENT_ZIP="agent.zip" \
286+
DEPS_JSON="deps.json" \
287+
./bin/build-stemcell-automation-zip.sh
288+
```
289+
290+
Use `make build` to build stembuild, providing the corresponding values for the stemcell version you
246291
would like to build, for example:
247292
```
248-
BOSH_AGENT_REPO=../bosh-agent STEMBUILD_VERSION=2019.70 make build
293+
STEMBUILD_VERSION=2019.70 make build
249294
```

‎bin/build-stemcell-automation-zip.sh

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env bash
2+
set -eu -o pipefail
3+
4+
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
5+
STEMCELL_AUTOMATION_PS1=$(ls "${ROOT_DIR}"/stemcell-automation/*ps1 | grep -iv Test)
6+
7+
: ${OPENSSH_ZIP?"Please see README.md on where to obtain this."}
8+
: ${BOSH_PSMODULES_ZIP?"Please see README.md on where to obtain this."}
9+
: ${AGENT_ZIP?"Please see README.md on how to construct this."}
10+
: ${DEPS_JSON?"Please see README.md on how to construct this."}
11+
12+
TEMP_DIR=$(mktemp -d)
13+
14+
cp "${OPENSSH_ZIP}" "${TEMP_DIR}/OpenSSH-Win64.zip"
15+
cp "${BOSH_PSMODULES_ZIP}" "${TEMP_DIR}/bosh-psmodules.zip"
16+
cp "${AGENT_ZIP}" "${TEMP_DIR}/agent.zip"
17+
cp "${DEPS_JSON}" "${TEMP_DIR}/deps.json"
18+
cp ${STEMCELL_AUTOMATION_PS1} "$TEMP_DIR"
19+
20+
rm "${ROOT_DIR}/assets/StemcellAutomation.zip"
21+
22+
zip -rj "${ROOT_DIR}/assets/StemcellAutomation.zip" "$TEMP_DIR"
23+
24+
rm -r "$TEMP_DIR"

‎stemcell-automation/generate-dep-json.bash

-25
This file was deleted.

0 commit comments

Comments
 (0)