From e501e0348870f64c84d82860c75face27f908317 Mon Sep 17 00:00:00 2001 From: Fabian Fulga Date: Tue, 18 Jun 2024 11:51:35 +0300 Subject: [PATCH] Adding cloud-init to images --- README.md | 12 +++-- internal/client/gcp.go | 4 +- internal/client/gcp_test.go | 95 ++++++++++++++++++++++++++++++++++++- internal/spec/spec.go | 29 +++-------- 4 files changed, 111 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index fb73771..756dc6f 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,9 @@ This will create a new Windows runner pool for the repo with ID `26ae13a1-13e9-4 **NOTE**: If you want to use a custom image that you created, specify the image name in the following format: `projects/my_project/global/images/my-custom-image` -Here is an example for a Linux pool that uses the image specified by its image name: +Always find a recent image to use. For example, to see available Windows server 2022 images, run something like `gcloud compute images list --filter windows-2022` or just search [here](https://console.cloud.google.com/compute/images). + +Linux pools support **ONLY** images with **CLOUD-INIT** already installed. Before using a linux pool, you must be sure that the image has cloud-init installed. Here is an example for a Linux pool that uses a custom image with cloud-init specified by its image name: ```bash garm-cli pool create \ @@ -78,14 +80,18 @@ garm-cli pool create \ --os-arch amd64 \ --enabled=true \ --flavor e2-medium \ - --image projects/debian-cloud/global/images/debian-11-bullseye-v20240110 \ + --image projects/garm-testing-424210/global/images/debian-cloud-init \ --min-idle-runners 0 \ --repo eb3f78b6-d667-4717-97c4-7aa1f3852138 \ --tags gcp,linux \ --provider-name gcp ``` -Always find a recent image to use. For example, to see available Windows server 2022 images, run something like `gcloud compute images list --filter windows-2022` or just search [here](https://console.cloud.google.com/compute/images). +**NOTE:** In order to [create a custom image](https://cloud.google.com/compute/docs/images/create-custom#create_image) with cloud-init, you have to *create an instance* with your desired Linux OS. Then connect to that instance and install `cloud-init`. After the install is finished, you can stop that instance and from the `Disk` of that instance, create a custom image. As example, if you use `projects/debian-cloud/global/images/debian-12-bookworm-v20240617`, you can install cloud-init on the instance like this: + +``` +sudo apt update && sudo apt install -y cloud-init +``` ## Tweaking the provider diff --git a/internal/client/gcp.go b/internal/client/gcp.go index 15d1480..6bb9ca2 100644 --- a/internal/client/gcp.go +++ b/internal/client/gcp.go @@ -35,7 +35,7 @@ import ( ) const ( - linuxStartupScript string = "startup-script" + linuxUserData string = "user-data" windowsStartupScript string = "sysprep-specialize-script-ps1" accessConfigType string = "ONE_TO_ONE_NAT" ) @@ -293,7 +293,7 @@ func selectStartupScript(osType params.OSType) string { case params.Windows: return windowsStartupScript case params.Linux: - return linuxStartupScript + return linuxUserData default: return "" } diff --git a/internal/client/gcp_test.go b/internal/client/gcp_test.go index 7193503..b6492df 100644 --- a/internal/client/gcp_test.go +++ b/internal/client/gcp_test.go @@ -33,7 +33,7 @@ import ( "google.golang.org/protobuf/proto" ) -func TestCreateInstance(t *testing.T) { +func TestCreateInstanceLinux(t *testing.T) { ctx := context.Background() mockClient := new(MockGcpClient) WaitOp = func(op *compute.Operation, ctx context.Context, opts ...gax.CallOption) error { @@ -53,6 +53,9 @@ func TestCreateInstance(t *testing.T) { mockOperation := &compute.Operation{} mockClient.On("Insert", mock.Anything, mock.Anything, mock.Anything).Return(mockOperation, nil) + spec.DefaultCloudConfigFunc = func(bootstrapParams params.BootstrapInstance, tools params.RunnerApplicationDownload, runnerName string) (string, error) { + return "MockUserData", nil + } spec := &spec.RunnerSpec{ Zone: "europe-west1-d", @@ -81,10 +84,100 @@ func TestCreateInstance(t *testing.T) { expectedInstance := &computepb.Instance{ Name: proto.String("garm-instance"), + Metadata: &computepb.Metadata{ + Items: []*computepb.Items{ + { + Key: proto.String("runner_name"), + Value: proto.String("garm-instance"), + }, + { + Key: proto.String(linuxUserData), + Value: proto.String("MockUserData"), + }, + }, + }, + } + result, err := gcpCli.CreateInstance(ctx, spec) + assert.NoError(t, err) + assert.Equal(t, expectedInstance.Name, result.Name) + for key, value := range expectedInstance.Metadata.Items { + assert.Equal(t, *expectedInstance.Metadata.Items[key].Key, *value.Key) + assert.Equal(t, *expectedInstance.Metadata.Items[key].Value, *value.Value) + } + mockClient.AssertExpectations(t) +} + +func TestCreateInstanceWindows(t *testing.T) { + ctx := context.Background() + mockClient := new(MockGcpClient) + WaitOp = func(op *compute.Operation, ctx context.Context, opts ...gax.CallOption) error { + return nil + } + gcpCli := &GcpCli{ + cfg: &config.Config{ + Zone: "europe-west1-d", + ProjectId: "my-project", + NetworkID: "my-network", + SubnetworkID: "my-subnetwork", + CredentialsFile: "path/to/credentials.json", + ExternalIPAccess: true, + }, + client: mockClient, + } + + mockOperation := &compute.Operation{} + mockClient.On("Insert", mock.Anything, mock.Anything, mock.Anything).Return(mockOperation, nil) + spec.DefaultRunnerInstallScriptFunc = func(bootstrapParams params.BootstrapInstance, tools params.RunnerApplicationDownload, runnerName string) ([]byte, error) { + return []byte("MockUserData"), nil + } + + spec := &spec.RunnerSpec{ + Zone: "europe-west1-d", + Tools: params.RunnerApplicationDownload{ + OS: proto.String("windows"), + Architecture: proto.String("amd64"), + DownloadURL: proto.String("MockURL"), + Filename: proto.String("garm-runner"), + }, + NetworkID: "my-network", + SubnetworkID: "my-subnetwork", + ControllerID: "my-controller", + NicType: "VIRTIO_NET", + DiskSize: 50, + CustomLabels: map[string]string{"key1": "value1"}, + NetworkTags: []string{"tag1", "tag2"}, + SourceSnapshot: "projects/garm-testing/global/snapshots/garm-snapshot", + BootstrapParams: params.BootstrapInstance{ + Name: "garm-instance", + Flavor: "n1-standard-1", + Image: "projects/garm-testing/global/images/garm-image", + OSType: params.Windows, + OSArch: "amd64", + }, + } + + expectedInstance := &computepb.Instance{ + Name: proto.String("garm-instance"), + Metadata: &computepb.Metadata{ + Items: []*computepb.Items{ + { + Key: proto.String("runner_name"), + Value: proto.String("garm-instance"), + }, + { + Key: proto.String(windowsStartupScript), + Value: proto.String("MockUserData"), + }, + }, + }, } result, err := gcpCli.CreateInstance(ctx, spec) assert.NoError(t, err) assert.Equal(t, expectedInstance.Name, result.Name) + for key, value := range expectedInstance.Metadata.Items { + assert.Equal(t, *expectedInstance.Metadata.Items[key].Key, *value.Key) + assert.Equal(t, *expectedInstance.Metadata.Items[key].Value, *value.Value) + } mockClient.AssertExpectations(t) } diff --git a/internal/spec/spec.go b/internal/spec/spec.go index d6903fb..00262b0 100644 --- a/internal/spec/spec.go +++ b/internal/spec/spec.go @@ -16,15 +16,12 @@ package spec import ( - "encoding/base64" "encoding/json" "fmt" "maps" "regexp" - "strings" "github.com/cloudbase/garm-provider-common/cloudconfig" - "github.com/cloudbase/garm-provider-common/defaults" "github.com/cloudbase/garm-provider-common/params" "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm-provider-gcp/config" @@ -104,6 +101,8 @@ const ( type ToolFetchFunc func(osType params.OSType, osArch params.OSArch, tools []params.RunnerApplicationDownload) (params.RunnerApplicationDownload, error) var DefaultToolFetch ToolFetchFunc = util.GetTools +var DefaultCloudConfigFunc = cloudconfig.GetCloudConfig +var DefaultRunnerInstallScriptFunc = cloudconfig.GetRunnerInstallScript func jsonSchemaValidation(schema json.RawMessage) error { schemaLoader := gojsonschema.NewStringLoader(jsonSchema) @@ -286,34 +285,18 @@ func (r RunnerSpec) ComposeUserData() (string, error) { switch r.BootstrapParams.OSType { case params.Linux: - udata, err := cloudconfig.GetRunnerInstallScript(r.BootstrapParams, r.Tools, r.BootstrapParams.Name) + // Get the cloud-init config + udata, err := DefaultCloudConfigFunc(bootstrapParams, r.Tools, bootstrapParams.Name) if err != nil { return "", fmt.Errorf("failed to generate userdata: %w", err) } + return udata, nil - asBase64 := base64.StdEncoding.EncodeToString(udata) - scriptCommands := []string{ - "sudo useradd -m " + defaults.DefaultUser + " || true", - // Create the runner home directory if it doesn't exist - "sudo mkdir -p /home/" + defaults.DefaultUser, - // Add user to sudoers - "sudo usermod -aG sudo " + defaults.DefaultUser, - // Check curl and tar are installed - "sudo apt-get update && sudo apt-get install -y curl tar", - // Install the runner - "echo " + asBase64 + " | base64 -d > /install_runner.sh", - "chmod +x /install_runner.sh", - "echo 'runner ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/garm", - "su -l -c /install_runner.sh " + defaults.DefaultUser, - } - script := strings.Join(scriptCommands, "\n") - return script, nil case params.Windows: - udata, err := cloudconfig.GetRunnerInstallScript(bootstrapParams, r.Tools, bootstrapParams.Name) + udata, err := DefaultRunnerInstallScriptFunc(bootstrapParams, r.Tools, bootstrapParams.Name) if err != nil { return "", fmt.Errorf("failed to generate userdata: %w", err) } - return string(udata), nil } return "", fmt.Errorf("unsupported OS type for cloud config: %s", r.BootstrapParams.OSType)