Merge branch 'main' into feat/scheduler-alert-system

Synthetixio · Aug 27, 2024 · 9ca75dd · 9ca75dd
2 parents 1017021 + fb57653
commit 9ca75dd
Show file tree

Hide file tree

Showing 128 changed files with 40,007 additions and 521 deletions.
diff --git a/.github/workflows/sql_linting.yml b/.github/workflows/sql_linting.yml
@@ -0,0 +1,22 @@
+name: Lint Transformer Models
+
+on:
+  push:
+    branches:
+      - main
+      - 'transformers/**'
+
+jobs:
+  lint-models:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 0
+      - uses: "actions/setup-python@v3"
+        with:
+            python-version: "3.10.11"
+      - name: Install SQLFluff
+        run: "pip install sqlfluff"
+      - name: Lint models
+        run: "diff-quality --violations=sqlfluff --compare-branch origin/main --fail-under 100"
diff --git a/.streamlit/secrets_example.toml b/.streamlit/secrets_example.toml
@@ -4,6 +4,7 @@ DB_USER = 'analytics'
 DB_PASS = 'analytics'
 DB_HOST = 'db'
 DB_PORT = '5432'
+DB_ENV = 'prod'
 
 [settings]
 SHOW_TESTNETS = 'false'

diff --git a/Makefile b/Makefile
@@ -18,6 +18,7 @@ build:
 	docker compose build transformer
 
 extract:
+	docker compose run extractors python main.py configs/eth_mainnet.yaml
 	docker compose run extractors python main.py configs/base_mainnet.yaml
 	docker compose run extractors python main.py configs/base_sepolia.yaml
 	docker compose run extractors python main.py configs/arbitrum_mainnet.yaml
@@ -32,5 +33,11 @@ import:
 dbt: build
 	docker compose run transformer dbt run --target prod --profiles-dir profiles --profile synthetix
 
+seed-prod: build
+	docker compose run transformer dbt seed --target prod --profiles-dir profiles --profile synthetix
+
+seed-dev: build
+	docker compose run transformer dbt seed --target dev --profiles-dir profiles --profile synthetix
+
 dbt-op: build
 	docker compose run transformer dbt run --target prod-op --profiles-dir profiles --profile synthetix
diff --git a/dashboard/About.py b/dashboard/About.py
@@ -62,6 +62,7 @@
 ]
 mainnet_pages = [
     Page(f"{PAGE_PREFIX}pages/All_Chains.py", "All Chains"),
+    Page(f"{PAGE_PREFIX}pages/Ethereum_Mainnet.py", "Ethereum"),
     Page(f"{PAGE_PREFIX}pages/Base_Mainnet.py", "Base"),
     Page(f"{PAGE_PREFIX}pages/Arbitrum_Mainnet.py", "Arbitrum"),
     Page(f"{PAGE_PREFIX}pages/Optimism_Mainnet.py", "Optimism"),

diff --git a/dashboard/modules/all_chains/core_stats.py b/dashboard/modules/all_chains/core_stats.py
@@ -38,8 +38,8 @@ def fetch_data(filters):
             apr_{resolution} as apr,
             apr_{resolution}_pnl as apr_pnl,
             apr_{resolution}_rewards as apr_rewards
-        FROM prod_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
-        LEFT JOIN prod_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+        FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
+        LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         
         UNION ALL
@@ -57,8 +57,8 @@ def fetch_data(filters):
             apr_{resolution} as apr,
             apr_{resolution}_pnl as apr_pnl,
             apr_{resolution}_rewards as apr_rewards
-        FROM prod_base_mainnet.fct_core_apr_base_mainnet apr
-        LEFT JOIN prod_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_apr_base_mainnet apr
+        LEFT JOIN {st.secrets.database.DB_ENV}_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         
         ORDER BY ts
@@ -80,8 +80,8 @@ def fetch_data(filters):
                 'Arbitrum' as label,
                 collateral_value,
                 cumulative_pnl
-            FROM prod_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
-            LEFT JOIN prod_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+            FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
+            LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
             WHERE ts >= '{start_date}' and ts <= '{end_date}'
         ) as a
         group by ts, label
@@ -98,8 +98,8 @@ def fetch_data(filters):
                 'Base' as label,
                 collateral_value,
                 cumulative_pnl
-            FROM prod_base_mainnet.fct_core_apr_base_mainnet apr
-            LEFT JOIN prod_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+            FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_apr_base_mainnet apr
+            LEFT JOIN {st.secrets.database.DB_ENV}_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
             WHERE ts >= '{start_date}' and ts <= '{end_date}'
         ) as b
         group by ts, label

diff --git a/dashboard/modules/all_chains/perp_stats.py b/dashboard/modules/all_chains/perp_stats.py
@@ -36,7 +36,7 @@ def fetch_data(filters):
                 trades,
                 exchange_fees as fees,
                 liquidated_accounts as liquidations
-            FROM prod_base_mainnet.fct_perp_stats_{resolution}_base_mainnet
+            FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_stats_{resolution}_base_mainnet
             WHERE ts >= '{start_date}' and ts <= '{end_date}'
         ),
         optimism as (
@@ -47,7 +47,7 @@ def fetch_data(filters):
                 trades,
                 exchange_fees + liquidation_fees as fees,
                 liquidations
-            FROM prod_optimism_mainnet.fct_v2_stats_{resolution}_optimism_mainnet
+            FROM {st.secrets.database.DB_ENV}_optimism_mainnet.fct_v2_stats_{resolution}_optimism_mainnet
             where ts >= '{filters["start_date"]}'
                 and ts <= '{filters["end_date"]}'
         )

diff --git a/dashboard/modules/arbitrum_mainnet/core_stats.py b/dashboard/modules/arbitrum_mainnet/core_stats.py
@@ -26,7 +26,7 @@ def fetch_data(filters):
     # get account data
     df_account_delegation = pd.read_sql_query(
         f"""
-        SELECT * FROM prod_arbitrum_mainnet.fct_core_account_delegation_arbitrum_mainnet
+        SELECT * FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_account_delegation_arbitrum_mainnet
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,
@@ -47,8 +47,8 @@ def fetch_data(filters):
             apr_{resolution} as apr,
             apr_{resolution}_pnl as apr_pnl,
             apr_{resolution}_rewards as apr_rewards
-        FROM prod_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
-        LEFT JOIN prod_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+        FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
+        LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         and pool_id = 1
         ORDER BY ts
@@ -66,8 +66,8 @@ def fetch_data(filters):
             collateral_value,
             rewards_usd,
             apr_{resolution}_rewards as apr_rewards
-        FROM prod_arbitrum_mainnet.fct_core_apr_rewards_arbitrum_mainnet apr
-        LEFT JOIN prod_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+        FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_apr_rewards_arbitrum_mainnet apr
+        LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         and pool_id = 1
         and apr.reward_token is not null

diff --git a/dashboard/modules/arbitrum_mainnet/spot_markets.py b/dashboard/modules/arbitrum_mainnet/spot_markets.py
@@ -32,7 +32,7 @@ def fetch_data(settings):
             tx_hash,
             synth_market_id,
             amount_wrapped
-        FROM prod_arbitrum_mainnet.fct_spot_wrapper_arbitrum_mainnet
+        FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_spot_wrapper_arbitrum_mainnet
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,
@@ -47,7 +47,7 @@ def fetch_data(settings):
             synth_market_id,
             amount,
             price
-        FROM prod_arbitrum_mainnet.fct_spot_atomics_arbitrum_mainnet
+        FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_spot_atomics_arbitrum_mainnet
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,
@@ -59,7 +59,7 @@ def fetch_data(settings):
             ts,
             synth_market_id,
             supply
-        FROM prod_arbitrum_mainnet.fct_synth_supply_arbitrum_mainnet
+        FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_synth_supply_arbitrum_mainnet
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,

diff --git a/dashboard/modules/arbitrum_sepolia/core_stats.py b/dashboard/modules/arbitrum_sepolia/core_stats.py
@@ -26,7 +26,7 @@ def fetch_data(filters):
     # get account data
     df_account_delegation = pd.read_sql_query(
         f"""
-        SELECT * FROM prod_arbitrum_sepolia.fct_core_account_delegation_arbitrum_sepolia
+        SELECT * FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_core_account_delegation_arbitrum_sepolia
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,
@@ -47,8 +47,8 @@ def fetch_data(filters):
             apr_{resolution} as apr,
             apr_{resolution}_pnl as apr_pnl,
             apr_{resolution}_rewards as apr_rewards
-        FROM prod_arbitrum_sepolia.fct_core_apr_arbitrum_sepolia apr
-        LEFT JOIN prod_seeds.arbitrum_sepolia_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+        FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_core_apr_arbitrum_sepolia apr
+        LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_sepolia_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         and pool_id = 1
         ORDER BY ts

diff --git a/dashboard/modules/arbitrum_sepolia/spot_markets.py b/dashboard/modules/arbitrum_sepolia/spot_markets.py
@@ -32,7 +32,7 @@ def fetch_data(settings):
             tx_hash,
             synth_market_id,
             amount_wrapped
-        FROM prod_arbitrum_sepolia.fct_spot_wrapper_arbitrum_sepolia
+        FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_spot_wrapper_arbitrum_sepolia
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,
@@ -47,7 +47,7 @@ def fetch_data(settings):
             synth_market_id,
             amount,
             price
-        FROM prod_arbitrum_sepolia.fct_spot_atomics_arbitrum_sepolia
+        FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_spot_atomics_arbitrum_sepolia
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,
@@ -59,7 +59,7 @@ def fetch_data(settings):
             ts,
             synth_market_id,
             supply
-        FROM prod_arbitrum_sepolia.fct_synth_supply_arbitrum_sepolia
+        FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_synth_supply_arbitrum_sepolia
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,

diff --git a/dashboard/modules/base_mainnet/core_stats.py b/dashboard/modules/base_mainnet/core_stats.py
@@ -26,7 +26,7 @@ def fetch_data(filters):
     # get account data
     df_account_delegation = pd.read_sql_query(
         f"""
-        SELECT * FROM prod_base_mainnet.fct_core_account_delegation_base_mainnet
+        SELECT * FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_account_delegation_base_mainnet
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,
@@ -47,8 +47,8 @@ def fetch_data(filters):
             apr_{resolution} as apr,
             apr_{resolution}_pnl as apr_pnl,
             apr_{resolution}_rewards as apr_rewards
-        FROM prod_base_mainnet.fct_core_apr_base_mainnet apr
-        LEFT JOIN prod_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_apr_base_mainnet apr
+        LEFT JOIN {st.secrets.database.DB_ENV}_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         and pool_id = 1
         ORDER BY ts
@@ -66,8 +66,8 @@ def fetch_data(filters):
             collateral_value,
             rewards_usd,
             apr_{resolution}_rewards as apr_rewards
-        FROM prod_base_mainnet.fct_core_apr_rewards_base_mainnet apr
-        LEFT JOIN prod_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_apr_rewards_base_mainnet apr
+        LEFT JOIN {st.secrets.database.DB_ENV}_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         and pool_id = 1
         and apr.reward_token is not null
@@ -147,9 +147,10 @@ def make_charts(data, filters):
         "apr": chart_lines(
             data["apr"],
             "ts",
-            ["apr", "apr_pnl", "apr_rewards"],
+            "apr",
             f"APR - {resolution} average",
             y_format="%",
+            color="collateral_type",
         ),
         "apr_token": chart_lines(
             data["apr_token"],

diff --git a/dashboard/modules/base_mainnet/perp_account.py b/dashboard/modules/base_mainnet/perp_account.py
@@ -28,7 +28,7 @@ def fetch_data(filters):
     # read data
     df_order = pd.read_sql_query(
         f"""
-        SELECT distinct account_id, sender FROM prod_base_mainnet.fct_perp_orders_base_mainnet
+        SELECT distinct account_id, sender FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_orders_base_mainnet
     """,
         db,
     )
@@ -40,7 +40,7 @@ def fetch_data(filters):
             market_id,
             acceptable_price,
             commitment_time
-        FROM prod_base_mainnet.fct_perp_previous_order_expired_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_previous_order_expired_base_mainnet
         WHERE account_id = {account_id if account_id else 'NULL'}
         and date(block_timestamp) >= '{start_date}' and date(block_timestamp) <= '{end_date}'
     """,
@@ -61,7 +61,7 @@ def fetch_data(filters):
             total_fees,
             accrued_funding,
             tracking_code
-        FROM prod_base_mainnet.fct_perp_trades_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_trades_base_mainnet
         WHERE account_id = '{account_id}'
         and ts >= '{start_date}' and ts <= '{end_date}'
     """,
@@ -74,7 +74,7 @@ def fetch_data(filters):
             cast(account_id as text) as account_id,
             synth_market_id,
             amount_delta
-        FROM prod_base_mainnet.fct_perp_collateral_modified_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_collateral_modified_base_mainnet
         WHERE account_id = {account_id if account_id else 'NULL'}
         and date(block_timestamp) >= '{start_date}' and date(block_timestamp) <= '{end_date}'
     """,
@@ -87,7 +87,7 @@ def fetch_data(filters):
             transaction_hash,
             cast(account_id as text) as account_id,
             interest
-        FROM prod_base_mainnet.fct_perp_interest_charged_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_interest_charged_base_mainnet
         WHERE account_id = {account_id if account_id else 'NULL'}
         and date(block_timestamp) >= '{start_date}' and date(block_timestamp) <= '{end_date}'
     """,
@@ -100,7 +100,7 @@ def fetch_data(filters):
             ts,
             account_id,
             total_reward
-        FROM prod_base_mainnet.fct_perp_liq_account_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_liq_account_base_mainnet
         WHERE account_id = '{account_id}'
         and ts >= '{start_date}' and ts <= '{end_date}'
     """,
@@ -112,7 +112,7 @@ def fetch_data(filters):
             ts,
             cumulative_volume,
             cumulative_fees
-        FROM prod_base_mainnet.fct_perp_account_stats_hourly_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_account_stats_hourly_base_mainnet
         WHERE account_id = '{account_id}'
         and ts >= '{start_date}' and ts <= '{end_date}'
         order by ts

diff --git a/dashboard/modules/base_mainnet/perp_integrators.py b/dashboard/modules/base_mainnet/perp_integrators.py
@@ -40,7 +40,7 @@ def fetch_data(filters):
             exchange_fees_share,
             referral_fees,
             referral_fees_share
-        FROM prod_base_mainnet.fct_perp_tracking_stats_{resolution}_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_tracking_stats_{resolution}_base_mainnet
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
     """,
         db,

diff --git a/dashboard/modules/base_mainnet/perp_keepers.py b/dashboard/modules/base_mainnet/perp_keepers.py
@@ -36,7 +36,7 @@ def fetch_data(filters):
             amount_settled_pct,
             settlement_rewards,
             settlement_rewards_pct
-        FROM prod_base_mainnet.fct_perp_keeper_stats_{resolution}_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_keeper_stats_{resolution}_base_mainnet
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         ORDER BY ts
         """,

diff --git a/dashboard/modules/base_mainnet/perp_markets.py b/dashboard/modules/base_mainnet/perp_markets.py
@@ -38,7 +38,7 @@ def fetch_data(filters):
             size_usd,
             short_oi_pct,
             long_oi_pct
-        FROM prod_base_mainnet.fct_perp_market_history_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_market_history_base_mainnet
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         ORDER BY ts
         """,
@@ -54,7 +54,7 @@ def fetch_data(filters):
             trades,
             exchange_fees,
             liquidations
-        FROM prod_base_mainnet.fct_perp_market_stats_daily_base_mainnet
+        FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_market_stats_daily_base_mainnet
         WHERE ts >= '{start_date}' and ts <= '{end_date}'
         """,
         db,