From 69a03bc5f5a10d62b4b988cd59a9c28f3b15b542 Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Fri, 9 Feb 2024 12:08:45 -0400 Subject: [PATCH 1/7] build: re-order Tiltfile --- dev/Tiltfile | 799 +++++++++++++++++++++++++-------------------------- 1 file changed, 388 insertions(+), 411 deletions(-) diff --git a/dev/Tiltfile b/dev/Tiltfile index f02b9a52f8..b54593c041 100644 --- a/dev/Tiltfile +++ b/dev/Tiltfile @@ -2,25 +2,10 @@ config.define_string_list("test") config.define_bool("bats") cfg = config.parse() -CORE_TEST_LABEL = "core" -CONSENT_TEST_LABEL = "consent" -DASHBOARD_TEST_LABEL = "dashboard" -PAY_TEST_LABEL = "pay" -ADMIN_PANEL_TEST_LABEL = "admin-panel" -MAP_TEST_LABEL = "map" - -TEST_RESOURCES = { - CORE_TEST_LABEL: "test-api", - CONSENT_TEST_LABEL: "test-consent", - DASHBOARD_TEST_LABEL: "test-dashboard", - PAY_TEST_LABEL: "test-pay", - ADMIN_PANEL_TEST_LABEL: "test-admin-panel", - MAP_TEST_LABEL: "test-map" -} - is_ci=("ci" in sys.argv) or cfg.get("bats", False) run_apps=not cfg.get("bats", False) + # From the Tilt docs: # # > By default, Tilt will not let you develop against a remote cluster. @@ -34,6 +19,11 @@ run_apps=not cfg.get("bats", False) # - https://docs.tilt.dev/api.html#api.allow_k8s_contexts allow_k8s_contexts(k8s_context()) + +# BUCK2 FILE DEPENDENCIES +# --- +# Needed for file monitoring to re-trigger Tilt-monitored resources + def _buck2_dep_inputs(target): cmd = [ "buck2", @@ -44,6 +34,295 @@ def _buck2_dep_inputs(target): return file_paths + +# INITIALIZATION TASKS +# --- + +local_resource( + name='hydra-dashboard', + labels = ['apps'], + cmd=[ + 'buck2', + 'run', + '//dev:setup-hydra-client', + '--', + 'dashboard', + 'authorization_code,refresh_token', + 'http://localhost:3001/api/auth/callback/blink', + ], + allow_parallel = True, + auto_init = run_apps, + resource_deps = [ + "hydra", + "api", + ] +) + +local_resource( + name='hydra-consent', + labels = ['auth'], + cmd=[ + 'buck2', + 'run', + '//dev:setup-hydra-client', + '--', + 'consent', + 'authorization_code,refresh_token', + 'http://localhost:3000', + ], + allow_parallel = True, + auto_init = run_apps, + resource_deps = [ + "hydra", + "api", + ] +) + +local_resource( + name='init-onchain', + labels = ['bitcoin'], + cmd='buck2 run //dev:init-onchain', + allow_parallel = True, + resource_deps = [ + "bitcoind", + "bria", + ] +) + +local_resource( + name='init-lightning', + labels = ['bitcoin'], + cmd='buck2 run //dev:init-lightning', + allow_parallel = True, + resource_deps = [ + "init-onchain", + "lnd1", + "lnd-outside-1", + "lnd-outside-2", + ] +) + + +# SERVE DEPENDENCIES +# --- + +callback_target = "//bats/helpers/callback:run" +local_resource( + "callback", + labels = ["test"], + serve_cmd = "buck2 run {}".format(callback_target), + allow_parallel = True, + deps = _buck2_dep_inputs(callback_target), +) + + +# SERVE RESOURCES +# --- + +core_serve_env = { + "HELMREVISION": "dev", + "NETWORK": "regtest", + "OATHKEEPER_DECISION_ENDPOINT": "http://localhost:4456", + "TWILIO_ACCOUNT_SID": "AC_twilio_id", + "TWILIO_AUTH_TOKEN": "AC_twilio_auth_token", + "TWILIO_VERIFY_SERVICE_ID": "VA_twilio_service", + "KRATOS_PG_CON": "postgres://dbuser:secret@localhost:5432/default?sslmode=disable", + "KRATOS_PUBLIC_API": "http://localhost:4433", + "KRATOS_ADMIN_API": "http://localhost:4434", + "KRATOS_MASTER_USER_PASSWORD": "passwordHardtoFindWithNumber123", + "PRICE_HOST": "localhost", + "PRICE_HISTORY_HOST": "localhost", + "BRIA_HOST": "localhost", + "BRIA_API_KEY": "bria_dev_000000000000000000000", + "NOTIFICATIONS_HOST": "localhost", + "MONGODB_CON": "mongodb://localhost:27017/galoy", + "REDIS_MASTER_NAME": "mymaster", + "REDIS_PASSWORD": "", + "REDIS_0_DNS": "localhost", + "REDIS_0_PORT": "6379", + "REDIS_TYPE": "standalone", + "UNSECURE_IP_FROM_REQUEST_OBJECT": "true", + "UNSECURE_DEFAULT_LOGIN_CODE": "000000", + "GEETEST_ID": "geetest_id", + "GEETEST_KEY": "geetest_key", + + "LND1_TLS": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNZVENDQWdlZ0F3SUJBZ0lSQU9zZzdYWFR4cnVZYlhkeTY2d3RuN1F3Q2dZSUtvWkl6ajBFQXdJd09ERWYKTUIwR0ExVUVDaE1XYkc1a0lHRjFkRzluWlc1bGNtRjBaV1FnWTJWeWRERVZNQk1HQTFVRUF4TU1PRFl4T1RneApNak5tT0Roak1CNFhEVEl6TURFeE9USXdOREUxTTFvWERUTTBNRGN5TVRJd05ERTFNMW93T0RFZk1CMEdBMVVFCkNoTVdiRzVrSUdGMWRHOW5aVzVsY21GMFpXUWdZMlZ5ZERFVk1CTUdBMVVFQXhNTU9EWXhPVGd4TWpObU9EaGoKTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFM1lieUlKWU1Vcm8zZkl0UFFucysxZ2lpTXI5NQpJUXRmclFDQ2JhOWVtcjI4TENmbk1vYy9VQVFwUlg3QVlvVFRneUdiMFBuZGNUODF5ZVgvYTlPa0RLT0I4VENCCjdqQU9CZ05WSFE4QkFmOEVCQU1DQXFRd0V3WURWUjBsQkF3d0NnWUlLd1lCQlFVSEF3RXdEd1lEVlIwVEFRSC8KQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVL1AxRHpJUkRzTEhHMU10d3NrZE5nZ0lub1Mwd2daWUdBMVVkRVFTQgpqakNCaTRJTU9EWXhPVGd4TWpObU9EaGpnZ2xzYjJOaGJHaHZjM1NDRFd4dVpDMXZkWFJ6YVdSbExUR0NEV3h1ClpDMXZkWFJ6YVdSbExUS0NEV3h1WkMxdmRYUnphV1JsTFRPQ0JHeHVaREdDQkd4dVpES0NCSFZ1YVhpQ0NuVnUKYVhod1lXTnJaWFNDQjJKMVptTnZibTZIQkg4QUFBR0hFQUFBQUFBQUFBQUFBQUFBQUFBQUFBR0hCS3dUQUJBdwpDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWhBSU5DNlJWQ3d6SzFYRnFxeVNLY0Y4QzQ5ZFlSOThjemdLNVdkcmNOCkxYYWlBaUJHYmtWeGhaeHdDaDVLQ1o1Z2M1Q2FsQ0RvaGNxVkdiaHNya0hHTFhpdHN3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", + "LND1_MACAROON": "AgEDbG5kAvgBAwoQB1FdhGa9xoewc1LEXmnURRIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYgqHDdwGCqx0aQL1/Z3uUfzCpeBhfapGf9s/AZPOVwf6s=", + "LND1_PUBKEY":"03ca1907342d5d37744cb7038375e1867c24a87564c293157c95b2a9d38dcfb4c2", + "LND1_DNS": "localhost", + "LND1_RPCPORT": "10009", + "LND1_NAME": "lnd1", + "LND1_TYPE": "offchain,onchain", + + "LND2_TLS": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNZVENDQWdlZ0F3SUJBZ0lSQU9zZzdYWFR4cnVZYlhkeTY2d3RuN1F3Q2dZSUtvWkl6ajBFQXdJd09ERWYKTUIwR0ExVUVDaE1XYkc1a0lHRjFkRzluWlc1bGNtRjBaV1FnWTJWeWRERVZNQk1HQTFVRUF4TU1PRFl4T1RneApNak5tT0Roak1CNFhEVEl6TURFeE9USXdOREUxTTFvWERUTTBNRGN5TVRJd05ERTFNMW93T0RFZk1CMEdBMVVFCkNoTVdiRzVrSUdGMWRHOW5aVzVsY21GMFpXUWdZMlZ5ZERFVk1CTUdBMVVFQXhNTU9EWXhPVGd4TWpObU9EaGoKTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFM1lieUlKWU1Vcm8zZkl0UFFucysxZ2lpTXI5NQpJUXRmclFDQ2JhOWVtcjI4TENmbk1vYy9VQVFwUlg3QVlvVFRneUdiMFBuZGNUODF5ZVgvYTlPa0RLT0I4VENCCjdqQU9CZ05WSFE4QkFmOEVCQU1DQXFRd0V3WURWUjBsQkF3d0NnWUlLd1lCQlFVSEF3RXdEd1lEVlIwVEFRSC8KQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVL1AxRHpJUkRzTEhHMU10d3NrZE5nZ0lub1Mwd2daWUdBMVVkRVFTQgpqakNCaTRJTU9EWXhPVGd4TWpObU9EaGpnZ2xzYjJOaGJHaHZjM1NDRFd4dVpDMXZkWFJ6YVdSbExUR0NEV3h1ClpDMXZkWFJ6YVdSbExUS0NEV3h1WkMxdmRYUnphV1JsTFRPQ0JHeHVaREdDQkd4dVpES0NCSFZ1YVhpQ0NuVnUKYVhod1lXTnJaWFNDQjJKMVptTnZibTZIQkg4QUFBR0hFQUFBQUFBQUFBQUFBQUFBQUFBQUFBR0hCS3dUQUJBdwpDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWhBSU5DNlJWQ3d6SzFYRnFxeVNLY0Y4QzQ5ZFlSOThjemdLNVdkcmNOCkxYYWlBaUJHYmtWeGhaeHdDaDVLQ1o1Z2M1Q2FsQ0RvaGNxVkdiaHNya0hHTFhpdHN3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", + "LND2_MACAROON": "AgEDbG5kAvgBAwoQX0BxfhQTxLTiqaceBnGnfBIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYgMAKlr1HehfBpn2R5RPE2IuY9r/18QBeLZxYgRidpos4=", + "LND2_PUBKEY": "039341ef13e776dc1611502cf510110d9ac5cdc252141f5997adcfd72cef34c3a7", + "LND2_DNS": "localhost", + "LND2_RPCPORT": "10010", + "LND2_NAME": "lnd2", + "LND2_TYPE": "offchain", + + "SVIX_SECRET": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE2OTE2NzIwMTQsImV4cCI6MjAwNzAzMjAxNCwibmJmIjoxNjkxNjcyMDE0LCJpc3MiOiJzdml4LXNlcnZlciIsInN1YiI6Im9yZ18yM3JiOFlkR3FNVDBxSXpwZ0d3ZFhmSGlyTXUifQ.b9s0aWSisNdUNki4edabBEToLNSwjC9-AiJQr4J3y4E", + "SVIX_ENDPOINT": "http://localhost:8071", + "EXPORTER_PORT": "3003" +} + +api_target = "//core/api:dev" +if is_ci: + api_target = "//core/api:api" +local_resource( + "api", + labels = ["core"], + cmd = "buck2 build {}".format(api_target), + serve_cmd = "buck2 run {}".format(api_target), + serve_env = core_serve_env, + allow_parallel = True, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "healthz", + port = 4012, + ), + ), + deps = _buck2_dep_inputs(api_target), + resource_deps = [ + "price", + "init-onchain", + "init-lightning", + "lnd1", + "lnd2", + "redis", + "mongodb", + "oathkeeper", + "svix", + ] +) + +api_trigger_target = "//core/api-trigger:dev" +if is_ci: + api_trigger_target = "//core/api-trigger:api-trigger" +local_resource( + "api-trigger", + labels = ["core"], + cmd = "buck2 build {}".format(api_trigger_target), + serve_cmd = "buck2 run //dev:stoppable-trigger {}".format(api_trigger_target), + serve_env = core_serve_env, + allow_parallel = True, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "healthz", + port = 4012, + ), + ), + deps = _buck2_dep_inputs(api_trigger_target), + resource_deps = [ + "price", + "init-onchain", + "lnd1", + "lnd2", + "redis", + "mongodb", + "oathkeeper", + ] +) + +api_exporter_target = "//core/api-exporter:dev" +if is_ci: + api_exporter_target = "//core/api-exporter:api-exporter" +local_resource( + "api-exporter", + labels = ["core"], + cmd = "buck2 build {}".format(api_exporter_target), + serve_cmd = "buck2 run {}".format(api_exporter_target), + serve_env = core_serve_env, + allow_parallel = True, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "healthz", + port = 4012, + ), + ), + deps = _buck2_dep_inputs(api_exporter_target), + resource_deps = [ + "price", + "init-onchain", + "lnd1", + "lnd2", + "redis", + "mongodb", + "oathkeeper", + ] +) + +api_ws_server_target = "//core/api-ws-server:dev" +if is_ci: + api_ws_server_target = "//core/api-ws-server:api-ws-server" +local_resource( + "api-ws-server", + labels = ["core"], + cmd = "buck2 build {}".format(api_ws_server_target), + serve_cmd = "buck2 run {}".format(api_ws_server_target), + serve_env = core_serve_env, + allow_parallel = True, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "healthz", + port = 4012, + ), + ), + deps = _buck2_dep_inputs(api_ws_server_target), + resource_deps = [ + "price", + "init-onchain", + "lnd1", + "lnd2", + "redis", + "mongodb", + "oathkeeper", + ] +) + +notifications_target = "//core/notifications:notifications" +local_resource( + "notifications", + labels = ["core"], + cmd = "buck2 build {}".format(notifications_target), + serve_cmd = "buck2 run {}".format(notifications_target), + serve_env = { + "PG_CON": "postgres://user:password@localhost:5433/pg", + "NOTIFICATIONS_CONFIG": "../core/notifications/notifications.yml", + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4317", + }, + deps = _buck2_dep_inputs(notifications_target), + allow_parallel = True, + resource_deps = [ + "api", + "notifications-pg" + ] +) + +api_keys_target = "//core/api-keys:api-keys" +local_resource( + "api-keys", + labels = ["auth"], + cmd = "buck2 build {}".format(api_keys_target), + serve_cmd = "buck2 run {}".format(api_keys_target), + serve_env = { + "PG_CON": "postgres://user:password@localhost:5431/pg", + "API_KEYS_CONFIG": "../core/api-keys/api-keys.yml", + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4317", + }, + deps = _buck2_dep_inputs(api_keys_target), + allow_parallel = True, + resource_deps = [ + "api", + "api-keys-pg" + ] +) + dashboard_target = "//apps/dashboard:dev" if is_ci: dashboard_target = '//apps/dashboard:dashboard' @@ -153,129 +432,6 @@ local_resource( ], ) -local_resource( - name='hydra-dashboard', - labels = ['apps'], - cmd=[ - 'buck2', - 'run', - '//dev:setup-hydra-client', - '--', - 'dashboard', - 'authorization_code,refresh_token', - 'http://localhost:3001/api/auth/callback/blink', - ], - allow_parallel = True, - auto_init = run_apps, - resource_deps = [ - "hydra", - "api", - ] -) - -consent_test_target = "//apps/consent:test-integration" -local_resource( - "test-consent", - labels = ["test"], - auto_init = is_ci and CONSENT_TEST_LABEL in cfg.get("test", []), - cmd = "buck2 test {}".format(consent_test_target), - allow_parallel = True, - resource_deps = [ - "consent", - "init-test-user", - ], -) - -dashboard_test_target = "//apps/dashboard:test-integration" -local_resource( - "test-dashboard", - labels = ["test"], - auto_init = is_ci and DASHBOARD_TEST_LABEL in cfg.get("test", []), - cmd = "buck2 test {}".format(dashboard_test_target), - resource_deps = [ - "consent", - "dashboard", - "init-test-user", - ], -) - -pay_test_target = "//apps/pay:test-integration" -local_resource( - "test-pay", - labels = ["test"], - auto_init = is_ci and PAY_TEST_LABEL in cfg.get("test", []), - cmd = "buck2 test {}".format(pay_test_target), - resource_deps = [ - "api", - "pay", - "add-test-users-with-usernames", - ], -) - -admin_panel_test_target = "//apps/admin-panel:test-integration" -local_resource( - "test-admin-panel", - labels = ["test"], - auto_init = is_ci and ADMIN_PANEL_TEST_LABEL in cfg.get("test", []), - cmd = "buck2 test {}".format(admin_panel_test_target), - resource_deps = [ - "admin-panel", - ], -) - -map_test_target = "//apps/map:test-integration" -local_resource( - "test-map", - labels = ["test"], - auto_init = is_ci and MAP_TEST_LABEL in cfg.get("test", []), - cmd = "buck2 test {}".format(map_test_target), - resource_deps = [ - "map", - ], -) - -local_resource( - name='init-test-user', - labels = ['test'], - cmd='buck2 run //dev:init-user', - allow_parallel = True, - resource_deps = [ - "oathkeeper", - "kratos", - "api", - ] -) - -local_resource( - name='add-test-users-with-usernames', - labels = ['test'], - cmd='buck2 run //dev:add-test-users-with-usernames', - allow_parallel = True, - resource_deps = [ - "oathkeeper", - "kratos", - "api", - ] -) - -local_resource( - name='fund-user', - labels = ['test'], - cmd='buck2 run //dev:fund-user', - allow_parallel = True, - resource_deps = [ - "oathkeeper", - "kratos", - "api", - "init-onchain", - "init-test-user", - "api-trigger", - "stablesats", - "price", - ] -) - - consent_target = "//apps/consent:dev" if is_ci: consent_target = '//apps/consent:consent' @@ -298,262 +454,11 @@ local_resource( ], ) -local_resource( - name='hydra-consent', - labels = ['auth'], - cmd=[ - 'buck2', - 'run', - '//dev:setup-hydra-client', - '--', - 'consent', - 'authorization_code,refresh_token', - 'http://localhost:3000', - ], - allow_parallel = True, - auto_init = run_apps, - resource_deps = [ - "hydra", - "api", - ] -) - -core_serve_env = { - "HELMREVISION": "dev", - "NETWORK": "regtest", - "OATHKEEPER_DECISION_ENDPOINT": "http://localhost:4456", - "TWILIO_ACCOUNT_SID": "AC_twilio_id", - "TWILIO_AUTH_TOKEN": "AC_twilio_auth_token", - "TWILIO_VERIFY_SERVICE_ID": "VA_twilio_service", - "KRATOS_PG_CON": "postgres://dbuser:secret@localhost:5432/default?sslmode=disable", - "KRATOS_PUBLIC_API": "http://localhost:4433", - "KRATOS_ADMIN_API": "http://localhost:4434", - "KRATOS_MASTER_USER_PASSWORD": "passwordHardtoFindWithNumber123", - "PRICE_HOST": "localhost", - "PRICE_HISTORY_HOST": "localhost", - "BRIA_HOST": "localhost", - "BRIA_API_KEY": "bria_dev_000000000000000000000", - "NOTIFICATIONS_HOST": "localhost", - "MONGODB_CON": "mongodb://localhost:27017/galoy", - "REDIS_MASTER_NAME": "mymaster", - "REDIS_PASSWORD": "", - "REDIS_0_DNS": "localhost", - "REDIS_0_PORT": "6379", - "REDIS_TYPE": "standalone", - "UNSECURE_IP_FROM_REQUEST_OBJECT": "true", - "UNSECURE_DEFAULT_LOGIN_CODE": "000000", - "GEETEST_ID": "geetest_id", - "GEETEST_KEY": "geetest_key", - - "LND1_TLS": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNZVENDQWdlZ0F3SUJBZ0lSQU9zZzdYWFR4cnVZYlhkeTY2d3RuN1F3Q2dZSUtvWkl6ajBFQXdJd09ERWYKTUIwR0ExVUVDaE1XYkc1a0lHRjFkRzluWlc1bGNtRjBaV1FnWTJWeWRERVZNQk1HQTFVRUF4TU1PRFl4T1RneApNak5tT0Roak1CNFhEVEl6TURFeE9USXdOREUxTTFvWERUTTBNRGN5TVRJd05ERTFNMW93T0RFZk1CMEdBMVVFCkNoTVdiRzVrSUdGMWRHOW5aVzVsY21GMFpXUWdZMlZ5ZERFVk1CTUdBMVVFQXhNTU9EWXhPVGd4TWpObU9EaGoKTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFM1lieUlKWU1Vcm8zZkl0UFFucysxZ2lpTXI5NQpJUXRmclFDQ2JhOWVtcjI4TENmbk1vYy9VQVFwUlg3QVlvVFRneUdiMFBuZGNUODF5ZVgvYTlPa0RLT0I4VENCCjdqQU9CZ05WSFE4QkFmOEVCQU1DQXFRd0V3WURWUjBsQkF3d0NnWUlLd1lCQlFVSEF3RXdEd1lEVlIwVEFRSC8KQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVL1AxRHpJUkRzTEhHMU10d3NrZE5nZ0lub1Mwd2daWUdBMVVkRVFTQgpqakNCaTRJTU9EWXhPVGd4TWpObU9EaGpnZ2xzYjJOaGJHaHZjM1NDRFd4dVpDMXZkWFJ6YVdSbExUR0NEV3h1ClpDMXZkWFJ6YVdSbExUS0NEV3h1WkMxdmRYUnphV1JsTFRPQ0JHeHVaREdDQkd4dVpES0NCSFZ1YVhpQ0NuVnUKYVhod1lXTnJaWFNDQjJKMVptTnZibTZIQkg4QUFBR0hFQUFBQUFBQUFBQUFBQUFBQUFBQUFBR0hCS3dUQUJBdwpDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWhBSU5DNlJWQ3d6SzFYRnFxeVNLY0Y4QzQ5ZFlSOThjemdLNVdkcmNOCkxYYWlBaUJHYmtWeGhaeHdDaDVLQ1o1Z2M1Q2FsQ0RvaGNxVkdiaHNya0hHTFhpdHN3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", - "LND1_MACAROON": "AgEDbG5kAvgBAwoQB1FdhGa9xoewc1LEXmnURRIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYgqHDdwGCqx0aQL1/Z3uUfzCpeBhfapGf9s/AZPOVwf6s=", - "LND1_PUBKEY":"03ca1907342d5d37744cb7038375e1867c24a87564c293157c95b2a9d38dcfb4c2", - "LND1_DNS": "localhost", - "LND1_RPCPORT": "10009", - "LND1_NAME": "lnd1", - "LND1_TYPE": "offchain,onchain", - - "LND2_TLS": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNZVENDQWdlZ0F3SUJBZ0lSQU9zZzdYWFR4cnVZYlhkeTY2d3RuN1F3Q2dZSUtvWkl6ajBFQXdJd09ERWYKTUIwR0ExVUVDaE1XYkc1a0lHRjFkRzluWlc1bGNtRjBaV1FnWTJWeWRERVZNQk1HQTFVRUF4TU1PRFl4T1RneApNak5tT0Roak1CNFhEVEl6TURFeE9USXdOREUxTTFvWERUTTBNRGN5TVRJd05ERTFNMW93T0RFZk1CMEdBMVVFCkNoTVdiRzVrSUdGMWRHOW5aVzVsY21GMFpXUWdZMlZ5ZERFVk1CTUdBMVVFQXhNTU9EWXhPVGd4TWpObU9EaGoKTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFM1lieUlKWU1Vcm8zZkl0UFFucysxZ2lpTXI5NQpJUXRmclFDQ2JhOWVtcjI4TENmbk1vYy9VQVFwUlg3QVlvVFRneUdiMFBuZGNUODF5ZVgvYTlPa0RLT0I4VENCCjdqQU9CZ05WSFE4QkFmOEVCQU1DQXFRd0V3WURWUjBsQkF3d0NnWUlLd1lCQlFVSEF3RXdEd1lEVlIwVEFRSC8KQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVL1AxRHpJUkRzTEhHMU10d3NrZE5nZ0lub1Mwd2daWUdBMVVkRVFTQgpqakNCaTRJTU9EWXhPVGd4TWpObU9EaGpnZ2xzYjJOaGJHaHZjM1NDRFd4dVpDMXZkWFJ6YVdSbExUR0NEV3h1ClpDMXZkWFJ6YVdSbExUS0NEV3h1WkMxdmRYUnphV1JsTFRPQ0JHeHVaREdDQkd4dVpES0NCSFZ1YVhpQ0NuVnUKYVhod1lXTnJaWFNDQjJKMVptTnZibTZIQkg4QUFBR0hFQUFBQUFBQUFBQUFBQUFBQUFBQUFBR0hCS3dUQUJBdwpDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWhBSU5DNlJWQ3d6SzFYRnFxeVNLY0Y4QzQ5ZFlSOThjemdLNVdkcmNOCkxYYWlBaUJHYmtWeGhaeHdDaDVLQ1o1Z2M1Q2FsQ0RvaGNxVkdiaHNya0hHTFhpdHN3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", - "LND2_MACAROON": "AgEDbG5kAvgBAwoQX0BxfhQTxLTiqaceBnGnfBIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYgMAKlr1HehfBpn2R5RPE2IuY9r/18QBeLZxYgRidpos4=", - "LND2_PUBKEY": "039341ef13e776dc1611502cf510110d9ac5cdc252141f5997adcfd72cef34c3a7", - "LND2_DNS": "localhost", - "LND2_RPCPORT": "10010", - "LND2_NAME": "lnd2", - "LND2_TYPE": "offchain", - - "SVIX_SECRET": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE2OTE2NzIwMTQsImV4cCI6MjAwNzAzMjAxNCwibmJmIjoxNjkxNjcyMDE0LCJpc3MiOiJzdml4LXNlcnZlciIsInN1YiI6Im9yZ18yM3JiOFlkR3FNVDBxSXpwZ0d3ZFhmSGlyTXUifQ.b9s0aWSisNdUNki4edabBEToLNSwjC9-AiJQr4J3y4E", - "SVIX_ENDPOINT": "http://localhost:8071", - "EXPORTER_PORT": "3003" -} - -callback_target = "//bats/helpers/callback:run" -local_resource( - "callback", - labels = ["test"], - serve_cmd = "buck2 run {}".format(callback_target), - allow_parallel = True, - deps = _buck2_dep_inputs(callback_target), -) - -api_target = "//core/api:dev" -if is_ci: - api_target = "//core/api:api" -local_resource( - "api", - labels = ["core"], - cmd = "buck2 build {}".format(api_target), - serve_cmd = "buck2 run {}".format(api_target), - serve_env = core_serve_env, - allow_parallel = True, - readiness_probe = probe( - period_secs = 5, - http_get = http_get_action( - path = "healthz", - port = 4012, - ), - ), - deps = _buck2_dep_inputs(api_target), - resource_deps = [ - "price", - "init-onchain", - "init-lightning", - "lnd1", - "lnd2", - "redis", - "mongodb", - "oathkeeper", - "svix", - ] -) - -api_trigger_target = "//core/api-trigger:dev" -if is_ci: - api_trigger_target = "//core/api-trigger:api-trigger" -local_resource( - "api-trigger", - labels = ["core"], - cmd = "buck2 build {}".format(api_trigger_target), - serve_cmd = "buck2 run //dev:stoppable-trigger {}".format(api_trigger_target), - serve_env = core_serve_env, - allow_parallel = True, - readiness_probe = probe( - period_secs = 5, - http_get = http_get_action( - path = "healthz", - port = 4012, - ), - ), - deps = _buck2_dep_inputs(api_trigger_target), - resource_deps = [ - "price", - "init-onchain", - "lnd1", - "lnd2", - "redis", - "mongodb", - "oathkeeper", - ] -) - -api_exporter_target = "//core/api-exporter:dev" -if is_ci: - api_exporter_target = "//core/api-exporter:api-exporter" -local_resource( - "api-exporter", - labels = ["core"], - cmd = "buck2 build {}".format(api_exporter_target), - serve_cmd = "buck2 run {}".format(api_exporter_target), - serve_env = core_serve_env, - allow_parallel = True, - readiness_probe = probe( - period_secs = 5, - http_get = http_get_action( - path = "healthz", - port = 4012, - ), - ), - deps = _buck2_dep_inputs(api_exporter_target), - resource_deps = [ - "price", - "init-onchain", - "lnd1", - "lnd2", - "redis", - "mongodb", - "oathkeeper", - ] -) - -api_ws_server_target = "//core/api-ws-server:dev" -if is_ci: - api_ws_server_target = "//core/api-ws-server:api-ws-server" -local_resource( - "api-ws-server", - labels = ["core"], - cmd = "buck2 build {}".format(api_ws_server_target), - serve_cmd = "buck2 run {}".format(api_ws_server_target), - serve_env = core_serve_env, - allow_parallel = True, - readiness_probe = probe( - period_secs = 5, - http_get = http_get_action( - path = "healthz", - port = 4012, - ), - ), - deps = _buck2_dep_inputs(api_ws_server_target), - resource_deps = [ - "price", - "init-onchain", - "lnd1", - "lnd2", - "redis", - "mongodb", - "oathkeeper", - ] -) -notifications_target = "//core/notifications:notifications" -local_resource( - "notifications", - labels = ["core"], - cmd = "buck2 build {}".format(notifications_target), - serve_cmd = "buck2 run {}".format(notifications_target), - serve_env = { - "PG_CON": "postgres://user:password@localhost:5433/pg", - "NOTIFICATIONS_CONFIG": "../core/notifications/notifications.yml", - "OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4317", - }, - deps = _buck2_dep_inputs(notifications_target), - allow_parallel = True, - resource_deps = [ - "api", - "notifications-pg" - ] -) - -local_resource( - name='init-onchain', - labels = ['bitcoin'], - cmd='buck2 run //dev:init-onchain', - allow_parallel = True, - resource_deps = [ - "bitcoind", - "bria", - ] -) - -local_resource( - name='init-lightning', - labels = ['bitcoin'], - cmd='buck2 run //dev:init-lightning', - allow_parallel = True, - resource_deps = [ - "init-onchain", - "lnd1", - "lnd-outside-1", - "lnd-outside-2", - ] -) +# DOCKER RESOURCES +# --- -api_keys_target = "//core/api-keys:api-keys" -local_resource( - "api-keys", - labels = ["auth"], - cmd = "buck2 build {}".format(api_keys_target), - serve_cmd = "buck2 run {}".format(api_keys_target), - serve_env = { - "PG_CON": "postgres://user:password@localhost:5431/pg", - "API_KEYS_CONFIG": "../core/api-keys/api-keys.yml", - "OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4317", - }, - deps = _buck2_dep_inputs(api_keys_target), - allow_parallel = True, - resource_deps = [ - "api", - "api-keys-pg" - ] -) +docker_compose("./docker-compose.deps.yml", project_name = "galoy-dev") docker_groups = { "auth": [ @@ -597,24 +502,6 @@ docker_groups = { ], } -to_run = cfg.get("to-run", []) -if to_run != []: - enabled_resources = [] - for svc in to_run: - enabled_resources.append(svc) - config.set_enabled_resources(enabled_resources) - -to_test = cfg.get("test", []) -if to_test != []: - enabled_resources = [] - for label in to_test: - svc = TEST_RESOURCES.get(label) - if svc: - enabled_resources.append(svc) - config.set_enabled_resources(enabled_resources) - -docker_compose("./docker-compose.deps.yml", project_name = "galoy-dev") - for service in docker_groups["bitcoin"]: dc_resource(service, labels = ["bitcoin"]) for service in docker_groups["tracing"]: @@ -628,6 +515,87 @@ for service in docker_groups["price"]: for service in docker_groups["integration"]: dc_resource(service, labels = ["integration"]) + +# TEST RESOURCES +# --- + +CORE_TEST_LABEL = "core" +CONSENT_TEST_LABEL = "consent" +DASHBOARD_TEST_LABEL = "dashboard" +PAY_TEST_LABEL = "pay" +ADMIN_PANEL_TEST_LABEL = "admin-panel" +MAP_TEST_LABEL = "map" + +TEST_RESOURCES = { + CORE_TEST_LABEL: "test-api", + CONSENT_TEST_LABEL: "test-consent", + DASHBOARD_TEST_LABEL: "test-dashboard", + PAY_TEST_LABEL: "test-pay", + ADMIN_PANEL_TEST_LABEL: "test-admin-panel", + MAP_TEST_LABEL: "test-map" +} + +consent_test_target = "//apps/consent:test-integration" +local_resource( + "test-consent", + labels = ["test"], + auto_init = is_ci and CONSENT_TEST_LABEL in cfg.get("test", []), + cmd = "buck2 test {}".format(consent_test_target), + allow_parallel = True, + resource_deps = [ + "consent", + "init-test-user", + ], +) + +dashboard_test_target = "//apps/dashboard:test-integration" +local_resource( + "test-dashboard", + labels = ["test"], + auto_init = is_ci and DASHBOARD_TEST_LABEL in cfg.get("test", []), + cmd = "buck2 test {}".format(dashboard_test_target), + resource_deps = [ + "consent", + "dashboard", + "init-test-user", + ], +) + +pay_test_target = "//apps/pay:test-integration" +local_resource( + "test-pay", + labels = ["test"], + auto_init = is_ci and PAY_TEST_LABEL in cfg.get("test", []), + cmd = "buck2 test {}".format(pay_test_target), + resource_deps = [ + "api", + "pay", + "add-test-users-with-usernames", + ], +) + +admin_panel_test_target = "//apps/admin-panel:test-integration" +local_resource( + "test-admin-panel", + labels = ["test"], + auto_init = is_ci and ADMIN_PANEL_TEST_LABEL in cfg.get("test", []), + cmd = "buck2 test {}".format(admin_panel_test_target), + resource_deps = [ + "admin-panel", + ], +) + +map_test_target = "//apps/map:test-integration" +local_resource( + "test-map", + labels = ["test"], + auto_init = is_ci and MAP_TEST_LABEL in cfg.get("test", []), + cmd = "buck2 test {}".format(map_test_target), + resource_deps = [ + "map", + ], +) + api_test_target = "//core/api:test-integration" local_resource( "test-api", @@ -636,3 +604,12 @@ local_resource( cmd = "buck2 test {}".format(api_test_target), resource_deps = [res for sublist in docker_groups.values() for res in sublist] + ["notifications"] ) + +to_test = cfg.get("test", []) +if to_test != []: + enabled_resources = [] + for label in to_test: + svc = TEST_RESOURCES.get(label) + if svc: + enabled_resources.append(svc) + config.set_enabled_resources(enabled_resources) From 21a12cb47f2a8de9be83b34f42d51eb4017b86ea Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:24:30 -0400 Subject: [PATCH 2/7] fix: change back integration upload action to always --- .github/workflows/integration-test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 5bdb264286..4d9234f3af 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -46,14 +46,13 @@ jobs: echo "Prepared args: $ARGS" echo "args=$ARGS" >> "$GITHUB_OUTPUT" - name: Build/start deps and run tests via tilt - id: tilt_test if: steps.prepare_args.outputs.args != '' run: nix develop -c xvfb-run ./dev/bin/tilt-ci.sh ${{ steps.prepare_args.outputs.args }} - name: Rename Tilt log - if: steps.tilt_test.outcome == 'success' || steps.tilt_test.outcome == 'failure' + if: always() run: mv dev/.e2e-tilt.log dev/e2e-tilt.log - name: Upload Tilt log - if: steps.tilt_test.outcome == 'success' || steps.tilt_test.outcome == 'failure' + if: always() uses: actions/upload-artifact@v2 with: name: Tilt log From f43487a5e1d1a9cae3a828abb3e9dc23db090ea4 Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:53:00 -0400 Subject: [PATCH 3/7] fix: adjust readiness_probe's in Tiltfile --- dev/Tiltfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/Tiltfile b/dev/Tiltfile index b54593c041..87874704c1 100644 --- a/dev/Tiltfile +++ b/dev/Tiltfile @@ -212,7 +212,7 @@ local_resource( period_secs = 5, http_get = http_get_action( path = "healthz", - port = 4012, + port = 8888, ), ), deps = _buck2_dep_inputs(api_trigger_target), @@ -241,7 +241,7 @@ local_resource( period_secs = 5, http_get = http_get_action( path = "healthz", - port = 4012, + port = 3003, ), ), deps = _buck2_dep_inputs(api_exporter_target), From 4697a2c013ef694a287d4a810489523fe5510a4e Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:53:18 -0400 Subject: [PATCH 4/7] ci: add readiness_probe to consent --- dev/Tiltfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev/Tiltfile b/dev/Tiltfile index 87874704c1..02bd8e0605 100644 --- a/dev/Tiltfile +++ b/dev/Tiltfile @@ -443,6 +443,13 @@ local_resource( deps = _buck2_dep_inputs(consent_target), allow_parallel = True, auto_init = run_apps, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "/", + port = 3000, + ), + ), resource_deps = [ "hydra-consent", "apollo-router", From b5971d326e38817497e073f12f8fd1bb571b52ff Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Fri, 9 Feb 2024 14:15:15 -0400 Subject: [PATCH 5/7] ci: add readiness_probe to other apps --- dev/Tiltfile | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dev/Tiltfile b/dev/Tiltfile index 02bd8e0605..98b4465728 100644 --- a/dev/Tiltfile +++ b/dev/Tiltfile @@ -336,6 +336,13 @@ local_resource( "NEXTAUTH_SECRET": "secret", "PORT": "3001", }, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "/", + port = 3001, + ), + ), deps = _buck2_dep_inputs(dashboard_target), allow_parallel = True, auto_init = run_apps, @@ -370,6 +377,13 @@ local_resource( serve_cmd = "buck2 run {}".format(pay_target), env = pay_env, serve_env = pay_env, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "/", + port = 3002, + ), + ), deps = _buck2_dep_inputs(pay_target), allow_parallel = True, resource_deps = [ @@ -396,6 +410,13 @@ local_resource( serve_cmd = "buck2 run {}".format(admin_panel_target), env = admin_panel_env, serve_env = admin_panel_env, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "/", + port = 3004, + ), + ), deps = _buck2_dep_inputs(admin_panel_target), allow_parallel = True, resource_deps = [ @@ -421,6 +442,13 @@ local_resource( serve_cmd = "buck2 run {}".format(map_target), env = map_env, serve_env = map_env, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "/", + port = 3005, + ), + ), deps = _buck2_dep_inputs(map_target), allow_parallel = True, resource_deps = [ From e9cdb21cd65e2cde59502ecabc9f450d56cb658a Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Fri, 9 Feb 2024 14:15:34 -0400 Subject: [PATCH 6/7] build: add price-history-pg healthcheck --- dev/docker-compose.deps.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/docker-compose.deps.yml b/dev/docker-compose.deps.yml index 6c6866fe82..728adb9aad 100644 --- a/dev/docker-compose.deps.yml +++ b/dev/docker-compose.deps.yml @@ -44,6 +44,11 @@ services: - POSTGRES_USER=galoy-price-usr - POSTGRES_PASSWORD=galoy-price-pwd - POSTGRES_DB=galoy-price-history + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 5s + timeout: 30s + retries: 5 api-keys-pg: image: postgres:14.1 environment: From 4e2c1ff6e165bcc6167fba8e806384691b63a38f Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Fri, 9 Feb 2024 15:03:30 -0400 Subject: [PATCH 7/7] build: add separate build args for check action --- .github/workflows/check-code-and-unit-test.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-code-and-unit-test.yaml b/.github/workflows/check-code-and-unit-test.yaml index f62dd68c40..428cb3a479 100644 --- a/.github/workflows/check-code-and-unit-test.yaml +++ b/.github/workflows/check-code-and-unit-test.yaml @@ -23,6 +23,7 @@ jobs: id: prepare_args run: | ARGS="" + BUILD_ARGS="" cat < labels.json ${{ toJSON(github.event.pull_request.labels.*.name) }} @@ -40,21 +41,26 @@ jobs: case "$LABEL" in dashboard|consent|pay|admin-panel|map) ARGS+=" //apps/$LABEL:test" + BUILD_ARGS+=" //apps/$LABEL:node_modules" ;; core) ARGS+=" //core/api:test" + BUILD_ARGS+=" //core/api:prod_build" ;; api-keys|notifications) ARGS+=" //core/$LABEL:$LABEL" + BUILD_ARGS+=" //core/$LABEL:$LABEL" ;; esac done echo "Prepared args: $ARGS" + echo "Prepared build_args: $BUILD_ARGS" echo "args=$ARGS" >> "$GITHUB_OUTPUT" + echo "build_args=$BUILD_ARGS" >> "$GITHUB_OUTPUT" - name: Build via buck2 if: steps.prepare_args.outputs.args != '' - run: nix develop -c buck2 build ${{ steps.prepare_args.outputs.args }} + run: nix develop -c buck2 build ${{ steps.prepare_args.outputs.build_args }} - name: Run checks and tests via buck2 if: steps.prepare_args.outputs.args != '' run: nix develop -c buck2 test ${{ steps.prepare_args.outputs.args }}