Skip to content

Commit

Permalink
Merge branch 'main' into juho/do-not-merge
Browse files Browse the repository at this point in the history
  • Loading branch information
@jvmakine
jvmakine committed Dec 2, 2024
2 parents 4d4ac16 + c4bab5d commit 6a6ec2c
Show file tree
Hide file tree
Showing 362 changed files with 9,007 additions and 9,931 deletions.
50 changes: 38 additions & 12 deletions Justfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ WATCHEXEC_ARGS := "-d 1s -e proto -e go -e sql -f sqlc.yaml --ignore **/types.ft
RELEASE := "build/release"
VERSION := `git describe --tags --always | sed -e 's/^v//'`
TIMESTAMP := `date +%s`
SCHEMA_OUT := "backend/protos/xyz/block/ftl/v1/schema/schema.proto"
SCHEMA_OUT := "backend/protos/xyz/block/ftl/schema/v1/schema.proto"
ZIP_DIRS := "go-runtime/compile/build-template " + \
"go-runtime/compile/external-module-template " + \
"go-runtime/compile/main-work-template " + \
Expand All @@ -20,13 +20,29 @@ CONSOLE_ROOT := "frontend/console"
FRONTEND_OUT := CONSOLE_ROOT + "/dist/index.html"
EXTENSION_OUT := "frontend/vscode/dist/extension.js"
PROTOS_IN := "backend/protos"
PROTOS_OUT := "backend/protos/xyz/block/ftl/v1/console/console.pb.go " + \
"backend/protos/xyz/block/ftl/v1/ftl.pb.go " + \
"backend/protos/xyz/block/ftl/v1/schema/schema.pb.go " + \
CONSOLE_ROOT + "/src/protos/xyz/block/ftl/v1/console/console_pb.ts " + \
PROTOS_OUT := "backend/protos/xyz/block/ftl/console/v1/console.pb.go " + \
"backend/protos/xyz/block/ftl//v1/ftl.pb.go " + \
"backend/protos/xyz/block/ftl/schema/v1/schema.pb.go " + \
CONSOLE_ROOT + "/src/protos/xyz/block/ftl/console/v1/console_pb.ts " + \
CONSOLE_ROOT + "/src/protos/xyz/block/ftl/v1/ftl_pb.ts " + \
CONSOLE_ROOT + "/src/protos/xyz/block/ftl/v1/schema/runtime_pb.ts " + \
CONSOLE_ROOT + "/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts"
CONSOLE_ROOT + "/src/protos/xyz/block/ftl/schema/v1/schema_pb.ts"
# Configuration for building Docker images
DOCKER_IMAGES := '''
{
"controller": {
"extra_binaries": ["ftl"],
"extra_files": ["ftl-provisioner-config.toml"]
},
"provisioner": {
"extra_binaries": ["ftl-provisioner-cloudformation"],
"extra_files": ["ftl-provisioner-config.toml"]
},
"cron": {},
"http-ingress": {},
"runner": {},
"runner-jvm": {}
}
'''

_help:
@just -l
Expand Down Expand Up @@ -57,7 +73,7 @@ dev *args:
watchexec -r {{WATCHEXEC_ARGS}} -- "just build-sqlc && ftl dev --plain {{args}}"

# Build everything
build-all: build-protos-unconditionally build-backend build-frontend build-generate build-sqlc build-zips lsp-generate build-jvm build-language-plugins
build-all: build-protos-unconditionally build-backend build-frontend build-backend-tests build-generate build-sqlc build-zips lsp-generate build-jvm build-language-plugins

# Run "go generate" on all packages
build-generate:
Expand All @@ -73,6 +89,7 @@ build +tools: build-protos build-zips build-frontend
# But it will be included if it was already built
build-without-frontend +tools: build-protos build-zips
#!/bin/bash
set -euo pipefail
mkdir -p frontend/console/dist
touch frontend/console/dist/.phoney
shopt -s extglob
Expand All @@ -91,7 +108,7 @@ build-backend:

# Build all backend tests
build-backend-tests:
go test -run ^NONE -tags integration,infrastructure ./...
go test -run ^NONE -tags integration,infrastructure ./... > /dev/null


build-jvm *args:
Expand Down Expand Up @@ -176,9 +193,9 @@ build-protos:
# Generate .proto files from .go types.
go2proto:
@mk "{{SCHEMA_OUT}}" : cmd/go2proto internal/schema -- go2proto -o "{{SCHEMA_OUT}}" \
-O 'go_package="github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema;schemapb"' \
-O 'go_package="github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/schema/v1;schemapb"' \
-O 'java_multiple_files=true' \
xyz.block.ftl.v1.schema ./internal/schema.Schema && buf format -w && buf lint
xyz.block.ftl.schema.v1 ./internal/schema.Schema && buf format -w && buf lint

# Unconditionally rebuild protos
build-protos-unconditionally: lint-protos pnpm-install go2proto
Expand Down Expand Up @@ -299,15 +316,24 @@ build-docker name:
# Run docker compose up with all docker compose files
compose-up:
#!/bin/bash
set -eo pipefail
set -o pipefail
docker_compose_files="
-f docker-compose.yml
-f internal/dev/docker-compose.grafana.yml
-f internal/dev/docker-compose.mysql.yml
-f internal/dev/docker-compose.postgres.yml
-f internal/dev/docker-compose.redpanda.yml
-f internal/dev/docker-compose.registry.yml"

docker compose -p "ftl" $docker_compose_files up -d --wait
status=$?
if [ $status -ne 0 ] && [ -n "${CI-}" ]; then
# CI fails regularly due to network issues. Retry once.
echo "docker compose up failed, retrying in 3s..."
docker compose -p "ftl" $docker_compose_files up -d --wait
docker compose -p "ftl" $docker_compose_files up -d --wait
fi


# Run a Just command in the Helm charts directory
chart *args:
Expand Down
46 changes: 23 additions & 23 deletions backend/controller/admin/admin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,13 @@ func (s *AdminService) Ping(ctx context.Context, req *connect.Request[ftlv1.Ping
}

// ConfigList returns the list of configuration values, optionally filtered by module.
func (s *AdminService) ConfigList(ctx context.Context, req *connect.Request[ftlv1.ListConfigRequest]) (*connect.Response[ftlv1.ListConfigResponse], error) {
func (s *AdminService) ConfigList(ctx context.Context, req *connect.Request[ftlv1.ConfigListRequest]) (*connect.Response[ftlv1.ConfigListResponse], error) {
listing, err := s.cm.List(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list configs: %w", err)
}

configs := []*ftlv1.ListConfigResponse_Config{}
configs := []*ftlv1.ConfigListResponse_Config{}
for _, config := range listing {
module, ok := config.Module.Get()
if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module {
Expand All @@ -76,16 +76,16 @@ func (s *AdminService) ConfigList(ctx context.Context, req *connect.Request[ftlv
}
}

configs = append(configs, &ftlv1.ListConfigResponse_Config{
configs = append(configs, &ftlv1.ConfigListResponse_Config{
RefPath: ref,
Value: cv,
})
}
return connect.NewResponse(&ftlv1.ListConfigResponse{Configs: configs}), nil
return connect.NewResponse(&ftlv1.ConfigListResponse{Configs: configs}), nil
}

// ConfigGet returns the configuration value for a given ref string.
func (s *AdminService) ConfigGet(ctx context.Context, req *connect.Request[ftlv1.GetConfigRequest]) (*connect.Response[ftlv1.GetConfigResponse], error) {
func (s *AdminService) ConfigGet(ctx context.Context, req *connect.Request[ftlv1.ConfigGetRequest]) (*connect.Response[ftlv1.ConfigGetResponse], error) {
var value any
err := s.cm.Get(ctx, refFromConfigRef(req.Msg.GetRef()), &value)
if err != nil {
Expand All @@ -95,26 +95,26 @@ func (s *AdminService) ConfigGet(ctx context.Context, req *connect.Request[ftlv1
if err != nil {
return nil, fmt.Errorf("failed to marshal value: %w", err)
}
return connect.NewResponse(&ftlv1.GetConfigResponse{Value: vb}), nil
return connect.NewResponse(&ftlv1.ConfigGetResponse{Value: vb}), nil
}

func configProviderKey(p *ftlv1.ConfigProvider) configuration.ProviderKey {
if p == nil {
return ""
}
switch *p {
case ftlv1.ConfigProvider_CONFIG_INLINE:
case ftlv1.ConfigProvider_CONFIG_PROVIDER_INLINE:
return providers.InlineProviderKey
case ftlv1.ConfigProvider_CONFIG_ENVAR:
case ftlv1.ConfigProvider_CONFIG_PROVIDER_ENVAR:
return providers.EnvarProviderKey
case ftlv1.ConfigProvider_CONFIG_DB:
case ftlv1.ConfigProvider_CONFIG_PROVIDER_DB:
return providers.DatabaseConfigProviderKey
}
return ""
}

// ConfigSet sets the configuration at the given ref to the provided value.
func (s *AdminService) ConfigSet(ctx context.Context, req *connect.Request[ftlv1.SetConfigRequest]) (*connect.Response[ftlv1.SetConfigResponse], error) {
func (s *AdminService) ConfigSet(ctx context.Context, req *connect.Request[ftlv1.ConfigSetRequest]) (*connect.Response[ftlv1.ConfigSetResponse], error) {
err := s.validateAgainstSchema(ctx, false, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
if err != nil {
return nil, err
Expand All @@ -124,25 +124,25 @@ func (s *AdminService) ConfigSet(ctx context.Context, req *connect.Request[ftlv1
if err != nil {
return nil, fmt.Errorf("failed to set config: %w", err)
}
return connect.NewResponse(&ftlv1.SetConfigResponse{}), nil
return connect.NewResponse(&ftlv1.ConfigSetResponse{}), nil
}

// ConfigUnset unsets the config value at the given ref.
func (s *AdminService) ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetConfigRequest]) (*connect.Response[ftlv1.UnsetConfigResponse], error) {
func (s *AdminService) ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.ConfigUnsetRequest]) (*connect.Response[ftlv1.ConfigUnsetResponse], error) {
err := s.cm.Unset(ctx, refFromConfigRef(req.Msg.GetRef()))
if err != nil {
return nil, fmt.Errorf("failed to unset config: %w", err)
}
return connect.NewResponse(&ftlv1.UnsetConfigResponse{}), nil
return connect.NewResponse(&ftlv1.ConfigUnsetResponse{}), nil
}

// SecretsList returns the list of secrets, optionally filtered by module.
func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftlv1.ListSecretsRequest]) (*connect.Response[ftlv1.ListSecretsResponse], error) {
func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftlv1.SecretsListRequest]) (*connect.Response[ftlv1.SecretsListResponse], error) {
listing, err := s.sm.List(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list secrets: %w", err)
}
secrets := []*ftlv1.ListSecretsResponse_Secret{}
secrets := []*ftlv1.SecretsListResponse_Secret{}
for _, secret := range listing {
module, ok := secret.Module.Get()
if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module {
Expand All @@ -164,16 +164,16 @@ func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftl
return nil, fmt.Errorf("failed to marshal value for %s: %w", ref, err)
}
}
secrets = append(secrets, &ftlv1.ListSecretsResponse_Secret{
secrets = append(secrets, &ftlv1.SecretsListResponse_Secret{
RefPath: ref,
Value: sv,
})
}
return connect.NewResponse(&ftlv1.ListSecretsResponse{Secrets: secrets}), nil
return connect.NewResponse(&ftlv1.SecretsListResponse{Secrets: secrets}), nil
}

// SecretGet returns the secret value for a given ref string.
func (s *AdminService) SecretGet(ctx context.Context, req *connect.Request[ftlv1.GetSecretRequest]) (*connect.Response[ftlv1.GetSecretResponse], error) {
func (s *AdminService) SecretGet(ctx context.Context, req *connect.Request[ftlv1.SecretGetRequest]) (*connect.Response[ftlv1.SecretGetResponse], error) {
var value any
err := s.sm.Get(ctx, refFromConfigRef(req.Msg.GetRef()), &value)
if err != nil {
Expand All @@ -183,11 +183,11 @@ func (s *AdminService) SecretGet(ctx context.Context, req *connect.Request[ftlv1
if err != nil {
return nil, fmt.Errorf("failed to marshal value: %w", err)
}
return connect.NewResponse(&ftlv1.GetSecretResponse{Value: vb}), nil
return connect.NewResponse(&ftlv1.SecretGetResponse{Value: vb}), nil
}

// SecretSet sets the secret at the given ref to the provided value.
func (s *AdminService) SecretSet(ctx context.Context, req *connect.Request[ftlv1.SetSecretRequest]) (*connect.Response[ftlv1.SetSecretResponse], error) {
func (s *AdminService) SecretSet(ctx context.Context, req *connect.Request[ftlv1.SecretSetRequest]) (*connect.Response[ftlv1.SecretSetResponse], error) {
err := s.validateAgainstSchema(ctx, true, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
if err != nil {
return nil, err
Expand All @@ -197,16 +197,16 @@ func (s *AdminService) SecretSet(ctx context.Context, req *connect.Request[ftlv1
if err != nil {
return nil, fmt.Errorf("failed to set secret: %w", err)
}
return connect.NewResponse(&ftlv1.SetSecretResponse{}), nil
return connect.NewResponse(&ftlv1.SecretSetResponse{}), nil
}

// SecretUnset unsets the secret value at the given ref.
func (s *AdminService) SecretUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetSecretRequest]) (*connect.Response[ftlv1.UnsetSecretResponse], error) {
func (s *AdminService) SecretUnset(ctx context.Context, req *connect.Request[ftlv1.SecretUnsetRequest]) (*connect.Response[ftlv1.SecretUnsetResponse], error) {
err := s.sm.Unset(ctx, refFromConfigRef(req.Msg.GetRef()))
if err != nil {
return nil, fmt.Errorf("failed to unset secret: %w", err)
}
return connect.NewResponse(&ftlv1.UnsetSecretResponse{}), nil
return connect.NewResponse(&ftlv1.SecretUnsetResponse{}), nil
}

func refFromConfigRef(cr *ftlv1.ConfigRef) configuration.Ref {
Expand Down
16 changes: 8 additions & 8 deletions backend/controller/admin/admin_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func testAdminConfigs(

module := ""
includeValues := true
resp, err := admin.ConfigList(ctx, connect.NewRequest(&ftlv1.ListConfigRequest{
resp, err := admin.ConfigList(ctx, connect.NewRequest(&ftlv1.ConfigListRequest{
Module: &module,
IncludeValues: &includeValues,
}))
Expand All @@ -100,7 +100,7 @@ func testAdminConfigs(
Module: &module,
Name: entry.Ref.Name,
}
resp, err := admin.ConfigGet(ctx, connect.NewRequest(&ftlv1.GetConfigRequest{Ref: ref}))
resp, err := admin.ConfigGet(ctx, connect.NewRequest(&ftlv1.ConfigGetRequest{Ref: ref}))
assert.NoError(t, err)
assert.Equal(t, entry.Value, string(resp.Msg.Value))
}
Expand All @@ -119,7 +119,7 @@ func testAdminSecrets(

module := ""
includeValues := true
resp, err := admin.SecretsList(ctx, connect.NewRequest(&ftlv1.ListSecretsRequest{
resp, err := admin.SecretsList(ctx, connect.NewRequest(&ftlv1.SecretsListRequest{
Module: &module,
IncludeValues: &includeValues,
}))
Expand All @@ -135,7 +135,7 @@ func testAdminSecrets(
Module: &module,
Name: entry.Ref.Name,
}
resp, err := admin.SecretGet(ctx, connect.NewRequest(&ftlv1.GetSecretRequest{Ref: ref}))
resp, err := admin.SecretGet(ctx, connect.NewRequest(&ftlv1.SecretGetRequest{Ref: ref}))
assert.NoError(t, err)
assert.Equal(t, entry.Value, string(resp.Msg.Value))
}
Expand Down Expand Up @@ -235,8 +235,8 @@ func testSetConfig(t testing.TB, ctx context.Context, admin *AdminService, modul
configRef.Module = &module
}

_, err = admin.ConfigSet(ctx, connect.NewRequest(&ftlv1.SetConfigRequest{
Provider: ftlv1.ConfigProvider_CONFIG_INLINE.Enum(),
_, err = admin.ConfigSet(ctx, connect.NewRequest(&ftlv1.ConfigSetRequest{
Provider: ftlv1.ConfigProvider_CONFIG_PROVIDER_INLINE.Enum(),
Ref: configRef,
Value: buffer,
}))
Expand All @@ -254,8 +254,8 @@ func testSetSecret(t testing.TB, ctx context.Context, admin *AdminService, modul
configRef.Module = &module
}

_, err = admin.SecretSet(ctx, connect.NewRequest(&ftlv1.SetSecretRequest{
Provider: ftlv1.SecretProvider_SECRET_INLINE.Enum(),
_, err = admin.SecretSet(ctx, connect.NewRequest(&ftlv1.SecretSetRequest{
Provider: ftlv1.SecretProvider_SECRET_PROVIDER_INLINE.Enum(),
Ref: configRef,
Value: buffer,
}))
Expand Down
16 changes: 8 additions & 8 deletions backend/controller/admin/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,28 +20,28 @@ type Client interface {
Ping(ctx context.Context, req *connect.Request[ftlv1.PingRequest]) (*connect.Response[ftlv1.PingResponse], error)

// List configuration.
ConfigList(ctx context.Context, req *connect.Request[ftlv1.ListConfigRequest]) (*connect.Response[ftlv1.ListConfigResponse], error)
ConfigList(ctx context.Context, req *connect.Request[ftlv1.ConfigListRequest]) (*connect.Response[ftlv1.ConfigListResponse], error)

// Get a config value.
ConfigGet(ctx context.Context, req *connect.Request[ftlv1.GetConfigRequest]) (*connect.Response[ftlv1.GetConfigResponse], error)
ConfigGet(ctx context.Context, req *connect.Request[ftlv1.ConfigGetRequest]) (*connect.Response[ftlv1.ConfigGetResponse], error)

// Set a config value.
ConfigSet(ctx context.Context, req *connect.Request[ftlv1.SetConfigRequest]) (*connect.Response[ftlv1.SetConfigResponse], error)
ConfigSet(ctx context.Context, req *connect.Request[ftlv1.ConfigSetRequest]) (*connect.Response[ftlv1.ConfigSetResponse], error)

// Unset a config value.
ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetConfigRequest]) (*connect.Response[ftlv1.UnsetConfigResponse], error)
ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.ConfigUnsetRequest]) (*connect.Response[ftlv1.ConfigUnsetResponse], error)

// List secrets.
SecretsList(ctx context.Context, req *connect.Request[ftlv1.ListSecretsRequest]) (*connect.Response[ftlv1.ListSecretsResponse], error)
SecretsList(ctx context.Context, req *connect.Request[ftlv1.SecretsListRequest]) (*connect.Response[ftlv1.SecretsListResponse], error)

// Get a secret.
SecretGet(ctx context.Context, req *connect.Request[ftlv1.GetSecretRequest]) (*connect.Response[ftlv1.GetSecretResponse], error)
SecretGet(ctx context.Context, req *connect.Request[ftlv1.SecretGetRequest]) (*connect.Response[ftlv1.SecretGetResponse], error)

// Set a secret.
SecretSet(ctx context.Context, req *connect.Request[ftlv1.SetSecretRequest]) (*connect.Response[ftlv1.SetSecretResponse], error)
SecretSet(ctx context.Context, req *connect.Request[ftlv1.SecretSetRequest]) (*connect.Response[ftlv1.SecretSetResponse], error)

// Unset a secret.
SecretUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetSecretRequest]) (*connect.Response[ftlv1.UnsetSecretResponse], error)
SecretUnset(ctx context.Context, req *connect.Request[ftlv1.SecretUnsetRequest]) (*connect.Response[ftlv1.SecretUnsetResponse], error)
}

// ShouldUseLocalClient returns whether a local admin client should be used based on the admin service client and the endpoint.
Expand Down
Loading

0 comments on commit 6a6ec2c

Please sign in to comment.