From 3f487f27a67566116d01448498e2b5a2b1dfcc32 Mon Sep 17 00:00:00 2001 From: Jerome Gravel-Niquet Date: Mon, 4 Nov 2024 13:18:25 -0500 Subject: [PATCH 1/3] attempt at accepting customize from file --- deploy.rb | 17 ++++++--- internal/command/launch/sessions.go | 56 +++++++++++++++++++---------- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/deploy.rb b/deploy.rb index e918c67dd4..a200cf0e39 100755 --- a/deploy.rb +++ b/deploy.rb @@ -18,7 +18,8 @@ Dir.chdir("/usr/src/app") DEPLOY_NOW = !get_env("DEPLOY_NOW").nil? -DEPLOY_CUSTOMIZE = !get_env("NO_DEPLOY_CUSTOMIZE") +DEPLOY_CUSTOMIZE_PATH = get_env("DEPLOY_CUSTOMIZE_PATH") +DEPLOY_CUSTOMIZE = !get_env("NO_DEPLOY_CUSTOMIZE") || !DEPLOY_CUSTOMIZE_PATH.nil? DEPLOY_ONLY = !get_env("DEPLOY_ONLY").nil? CREATE_AND_PUSH_BRANCH = !get_env("DEPLOY_CREATE_AND_PUSH_BRANCH").nil? FLYIO_BRANCH_NAME = "flyio-new-files" @@ -239,15 +240,21 @@ if DEPLOY_CUSTOMIZE manifest = in_step Step::CUSTOMIZE do - cmd = "flyctl launch sessions create --session-path /tmp/session.json --manifest-path #{MANIFEST_PATH} --from-manifest #{MANIFEST_PATH}" + if DEPLOY_CUSTOMIZE_PATH.nil? + cmd = "flyctl launch sessions create --session-path /tmp/session.json --manifest-path #{MANIFEST_PATH} --from-manifest #{MANIFEST_PATH}" - exec_capture(cmd) - session = JSON.parse(File.read("/tmp/session.json")) + exec_capture(cmd) + session = JSON.parse(File.read("/tmp/session.json")) - artifact Artifact::SESSION, session + artifact Artifact::SESSION, session + end cmd = "flyctl launch sessions finalize --session-path /tmp/session.json --manifest-path #{MANIFEST_PATH}" + if !DEPLOY_CUSTOMIZE_PATH.nil? + cmd += " --from-file #{DEPLOY_CUSTOMIZE_PATH}" + end + exec_capture(cmd) manifest = JSON.parse(File.read("/tmp/manifest.json")) diff --git a/internal/command/launch/sessions.go b/internal/command/launch/sessions.go index b74b26279a..5dfabfce2e 100644 --- a/internal/command/launch/sessions.go +++ b/internal/command/launch/sessions.go @@ -94,6 +94,11 @@ func newSessions() *cobra.Command { Description: "Path to write the manifest info to", Default: "manifest.json", }, + flag.String{ + Name: "from-file", + Description: "Path to a CLI session JSON file", + Default: "", + }, ) // not that useful anywhere else yet @@ -192,14 +197,39 @@ func runSessionFinalize(ctx context.Context) (err error) { io := iostreams.FromContext(ctx) logger := logger.FromContext(ctx) - sessionBytes, err := os.ReadFile(flag.GetString(ctx, "session-path")) - if err != nil { - return err - } + var finalSession fly.CLISession - var session fly.CLISession - if err := json.Unmarshal(sessionBytes, &session); err != nil { - return err + if customizePath := flag.GetString(ctx, "from-file"); customizePath != "" { + sessionBytes, err := os.ReadFile(customizePath) + if err != nil { + return err + } + + if err := json.Unmarshal(sessionBytes, &finalSession); err != nil { + return err + } + } else { + sessionBytes, err := os.ReadFile(flag.GetString(ctx, "session-path")) + if err != nil { + return err + } + + var session fly.CLISession + if err := json.Unmarshal(sessionBytes, &session); err != nil { + return err + } + + // FIXME: better timeout here + ctx, cancel := context.WithTimeout(ctx, 15*time.Minute) + defer cancel() + + finalSession, err = waitForCLISession(ctx, logger, io.ErrOut, session.ID) + switch { + case errors.Is(err, context.DeadlineExceeded): + return errors.New("session expired, please try again") + case err != nil: + return err + } } manifestBytes, err := os.ReadFile(flag.GetString(ctx, "manifest-path")) @@ -219,18 +249,6 @@ func runSessionFinalize(ctx context.Context) (err error) { warnedNoCcHa: true, } - // FIXME: better timeout here - ctx, cancel := context.WithTimeout(ctx, 15*time.Minute) - defer cancel() - - finalSession, err := waitForCLISession(ctx, logger, io.ErrOut, session.ID) - switch { - case errors.Is(err, context.DeadlineExceeded): - return errors.New("session expired, please try again") - case err != nil: - return err - } - // Hack because somewhere from between UI and here, the numbers get converted to strings if err := patchNumbers(finalSession.Metadata, "vm_cpus", "vm_memory"); err != nil { return err From 6d565cff5c83e87d08f80f584527c8358f59f510 Mon Sep 17 00:00:00 2001 From: Jerome Gravel-Niquet Date: Mon, 4 Nov 2024 16:01:10 -0500 Subject: [PATCH 2/3] add a test for pre-customize.json --- test/deployer/deployer_test.go | 34 ++++++++++++++++++++++++++++++++++ test/testlib/deployer.go | 23 +++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/test/deployer/deployer_test.go b/test/deployer/deployer_test.go index cf65f8a8a3..d328f0ff69 100644 --- a/test/deployer/deployer_test.go +++ b/test/deployer/deployer_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/stretchr/testify/require" + "github.com/superfly/fly-go" "github.com/superfly/flyctl/test/testlib" ) @@ -198,6 +199,39 @@ func TestLaunchGoFromRepo(t *testing.T) { require.Contains(t, string(body), "I'm running in the yyz region") } +func TestLaunchPreCustomized(t *testing.T) { + customize := fly.CLISession{ + ID: "", + URL: "", + AccessToken: "", + Metadata: map[string]interface{}{ + "vm_memory": 2048, + }, + } + + deploy := testDeployer(t, + createRandomApp, + testlib.WithRegion("yyz"), + testlib.WithPreCustomize(&customize), + testlib.WithouExtensions, + testlib.DeployNow, + testlib.WithGitRepo("https://github.com/fly-apps/go-example"), + ) + + appName := deploy.Extra["appName"].(string) + + manifest, err := deploy.Output().ArtifactManifest() + require.NoError(t, err) + require.NotNil(t, manifest) + + require.Equal(t, manifest.Plan.Guest().MemoryMB, 2048) + + body, err := testlib.RunHealthCheck(fmt.Sprintf("https://%s.fly.dev", appName)) + require.NoError(t, err) + + require.Contains(t, string(body), "I'm running in the yyz region") +} + func TestLaunchRails70(t *testing.T) { deploy := testDeployer(t, withFixtureApp("deploy-rails-7.0"), diff --git a/test/testlib/deployer.go b/test/testlib/deployer.go index 22559361db..d13328d60f 100644 --- a/test/testlib/deployer.go +++ b/test/testlib/deployer.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "testing" @@ -21,6 +22,7 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/require" + "github.com/superfly/fly-go" "github.com/superfly/flyctl/internal/command/launch" ) @@ -94,6 +96,7 @@ type DeployTestRun struct { skipExtensions bool copyConfig bool optOutGha bool + customizePath string deployOnly bool deployNow bool @@ -127,6 +130,22 @@ func WithApp(app string) func(*DeployTestRun) { } } +func WithPreCustomize(customize *fly.CLISession) func(*DeployTestRun) { + b, err := json.Marshal(customize) + if err != nil { + panic(err) + } + return func(d *DeployTestRun) { + p := filepath.Join(d.WorkDir(), "customize.json") + if err := os.WriteFile(p, b, 0666); err != nil { + panic(err) + } + dst := "/opt/customize.json" + d.containerBinds = append(d.containerBinds, fmt.Sprintf("%s:%s", p, dst)) + d.customizePath = dst + } +} + func WithGitRepo(repo string) func(*DeployTestRun) { return func(d *DeployTestRun) { d.gitRepo = repo @@ -237,6 +256,10 @@ func (d *DeployTestRun) Start(ctx context.Context) error { env = append(env, "DEPLOYER_CLEANUP_BEFORE_EXIT=1") } + if d.customizePath != "" { + env = append(env, fmt.Sprintf("DEPLOY_CUSTOMIZE_PATH=%s", d.customizePath)) + } + fmt.Printf("creating container... image=%s\n", d.deployerImage) cont, err := d.dockerClient.ContainerCreate(ctx, &container.Config{ Image: d.deployerImage, From aede710e4ac1f3d0f557566054c499cbeaa9b035 Mon Sep 17 00:00:00 2001 From: Jerome Gravel-Niquet Date: Tue, 5 Nov 2024 10:18:40 -0500 Subject: [PATCH 3/3] customize json is just a map of data --- deploy.rb | 1 + internal/command/launch/launch.go | 18 ++++++++++++++++++ internal/command/launch/sessions.go | 26 ++++++++++++++------------ test/deployer/deployer_test.go | 11 +++-------- test/testlib/deployer.go | 3 +-- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/deploy.rb b/deploy.rb index a200cf0e39..9d851e280e 100755 --- a/deploy.rb +++ b/deploy.rb @@ -253,6 +253,7 @@ if !DEPLOY_CUSTOMIZE_PATH.nil? cmd += " --from-file #{DEPLOY_CUSTOMIZE_PATH}" + artifact Artifact::SESSION, JSON.parse(File.read(DEPLOY_CUSTOMIZE_PATH)) end exec_capture(cmd) diff --git a/internal/command/launch/launch.go b/internal/command/launch/launch.go index 21827b619c..ed8c4982d3 100644 --- a/internal/command/launch/launch.go +++ b/internal/command/launch/launch.go @@ -185,6 +185,24 @@ func updateConfig(plan *plan.LaunchPlan, env map[string]string, appConfig *appco appConfig.HTTPService = nil } appConfig.Compute = plan.Compute + + if plan.CPUKind != "" { + for _, c := range appConfig.Compute { + c.CPUKind = plan.CPUKind + } + } + + if plan.CPUs != 0 { + for _, c := range appConfig.Compute { + c.CPUs = plan.CPUs + } + } + + if plan.MemoryMB != 0 { + for _, c := range appConfig.Compute { + c.MemoryMB = plan.MemoryMB + } + } } // createApp creates the fly.io app for the plan diff --git a/internal/command/launch/sessions.go b/internal/command/launch/sessions.go index 5dfabfce2e..a6c9ede18e 100644 --- a/internal/command/launch/sessions.go +++ b/internal/command/launch/sessions.go @@ -197,7 +197,7 @@ func runSessionFinalize(ctx context.Context) (err error) { io := iostreams.FromContext(ctx) logger := logger.FromContext(ctx) - var finalSession fly.CLISession + var finalMeta map[string]interface{} if customizePath := flag.GetString(ctx, "from-file"); customizePath != "" { sessionBytes, err := os.ReadFile(customizePath) @@ -205,7 +205,7 @@ func runSessionFinalize(ctx context.Context) (err error) { return err } - if err := json.Unmarshal(sessionBytes, &finalSession); err != nil { + if err := json.Unmarshal(sessionBytes, &finalMeta); err != nil { return err } } else { @@ -223,13 +223,15 @@ func runSessionFinalize(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, 15*time.Minute) defer cancel() - finalSession, err = waitForCLISession(ctx, logger, io.ErrOut, session.ID) + finalSession, err := waitForCLISession(ctx, logger, io.ErrOut, session.ID) switch { case errors.Is(err, context.DeadlineExceeded): return errors.New("session expired, please try again") case err != nil: return err } + + finalMeta = finalSession.Metadata } manifestBytes, err := os.ReadFile(flag.GetString(ctx, "manifest-path")) @@ -250,14 +252,7 @@ func runSessionFinalize(ctx context.Context) (err error) { } // Hack because somewhere from between UI and here, the numbers get converted to strings - if err := patchNumbers(finalSession.Metadata, "vm_cpus", "vm_memory"); err != nil { - return err - } - - // Wasteful, but gets the job done without uprooting the session types. - // Just round-trip the map[string]interface{} back into json, so we can re-deserialize it into a complete type. - metaJson, err := json.Marshal(finalSession.Metadata) - if err != nil { + if err := patchNumbers(finalMeta, "vm_cpus", "vm_memory"); err != nil { return err } @@ -272,6 +267,13 @@ func runSessionFinalize(ctx context.Context) (err error) { oldPlan := helpers.Clone(state.Plan) + // Wasteful, but gets the job done without uprooting the session types. + // Just round-trip the map[string]interface{} back into json, so we can re-deserialize it into a complete type. + metaJson, err := json.Marshal(finalMeta) + if err != nil { + return err + } + err = json.Unmarshal(metaJson, &state.Plan) if err != nil { return err @@ -280,7 +282,7 @@ func runSessionFinalize(ctx context.Context) (err error) { // Patch in some fields that we keep in the plan that aren't persisted by the UI. // Technically, we should probably just be persisting this, but there's // no clear value to the UI having these fields currently. - if _, ok := finalSession.Metadata["ha"]; !ok { + if _, ok := finalMeta["ha"]; !ok { state.Plan.HighAvailability = oldPlan.HighAvailability } // This should never be changed by the UI!! diff --git a/test/deployer/deployer_test.go b/test/deployer/deployer_test.go index d328f0ff69..0a3f97f734 100644 --- a/test/deployer/deployer_test.go +++ b/test/deployer/deployer_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "github.com/superfly/fly-go" "github.com/superfly/flyctl/test/testlib" ) @@ -200,13 +199,8 @@ func TestLaunchGoFromRepo(t *testing.T) { } func TestLaunchPreCustomized(t *testing.T) { - customize := fly.CLISession{ - ID: "", - URL: "", - AccessToken: "", - Metadata: map[string]interface{}{ - "vm_memory": 2048, - }, + customize := map[string]interface{}{ + "vm_memory": 2048, } deploy := testDeployer(t, @@ -225,6 +219,7 @@ func TestLaunchPreCustomized(t *testing.T) { require.NotNil(t, manifest) require.Equal(t, manifest.Plan.Guest().MemoryMB, 2048) + require.Equal(t, manifest.Config.Compute[0].MemoryMB, 2048) body, err := testlib.RunHealthCheck(fmt.Sprintf("https://%s.fly.dev", appName)) require.NoError(t, err) diff --git a/test/testlib/deployer.go b/test/testlib/deployer.go index d13328d60f..04637e85fa 100644 --- a/test/testlib/deployer.go +++ b/test/testlib/deployer.go @@ -22,7 +22,6 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/require" - "github.com/superfly/fly-go" "github.com/superfly/flyctl/internal/command/launch" ) @@ -130,7 +129,7 @@ func WithApp(app string) func(*DeployTestRun) { } } -func WithPreCustomize(customize *fly.CLISession) func(*DeployTestRun) { +func WithPreCustomize(customize interface{}) func(*DeployTestRun) { b, err := json.Marshal(customize) if err != nil { panic(err)