Merge main

Synthetixio · Aug 27, 2024 · bf32e13 · bf32e13
2 parents c9cb2ac + fb57653
commit bf32e13
Show file tree

Hide file tree

Showing 7 changed files with 86 additions and 43 deletions.
diff --git a/.streamlit/secrets_example.toml b/.streamlit/secrets_example.toml
@@ -4,6 +4,7 @@ DB_USER = 'analytics'
 DB_PASS = 'analytics'
 DB_HOST = 'db'
 DB_PORT = '5432'
+DB_ENV = 'prod'
 
 [settings]
 SHOW_TESTNETS = 'false'

diff --git a/extractors/requirements.txt b/extractors/requirements.txt
@@ -5,4 +5,5 @@ duckdb==0.10.2
 polars-lts-cpu==1.1.0
 pandas
 numpy
-synthetix==0.1.13
+synthetix==0.1.13
+web3==6.20.2
diff --git a/scheduler/dags/v3_etl.py b/scheduler/dags/v3_etl.py
@@ -8,6 +8,7 @@
 # environment variables
 WORKING_DIR = os.getenv("WORKING_DIR")
 NETWORK_RPCS = {
+    "eth_mainnet": "NETWORK_1_RPC",
     "base_mainnet": "NETWORK_8453_RPC",
     "base_sepolia": "NETWORK_84532_RPC",
     "arbitrum_mainnet": "NETWORK_42161_RPC",
@@ -49,7 +50,7 @@ def create_docker_operator(dag, task_id, config_file, image, command, network_en
     )
 
 
-def create_dag(network, rpc_var, target='dev'):
+def create_dag(network, rpc_var, target="dev"):
     version = f"{network}_{target}"
 
     dag = DAG(
@@ -78,10 +79,10 @@ def create_dag(network, rpc_var, target='dev'):
         config_file=None,
         image="data-transformer",
         command=f"dbt test --target {target if network != 'optimism_mainnet' else target + '-op'} --select tag:{network} --profiles-dir profiles --profile synthetix",
-        network_env_var=rpc_var
+        network_env_var=rpc_var,
     )
 
-    if target == 'prod':
+    if target == "prod":
         extract_task_id = f"extract_{version}"
         config_file = f"configs/{network}.yaml"
         extract_task = create_docker_operator(
@@ -101,5 +102,5 @@ def create_dag(network, rpc_var, target='dev'):
 
 
 for network, rpc_var in NETWORK_RPCS.items():
-    for target in ['dev', 'prod']:
+    for target in ["dev", "prod"]:
         globals()[f"v3_etl_{network}_{target}"] = create_dag(network, rpc_var, target)
diff --git a/transformers/synthetix/models/marts/eth/mainnet/core/fct_pool_pnl_hourly_eth_mainnet.sql b/transformers/synthetix/models/marts/eth/mainnet/core/fct_pool_pnl_hourly_eth_mainnet.sql
@@ -5,13 +5,12 @@
 
 with dim as (
     select
-        p.pool_id,
-        p.collateral_type,
         generate_series(
             date_trunc('hour', min(t.ts)),
-            date_trunc('hour', max(t.ts)),
-            '1 hour'::INTERVAL
-        ) as ts
+            date_trunc('hour', max(t.ts)), '1 hour'::interval
+        ) as ts,
+        p.pool_id,
+        p.collateral_type
     from
         (
             select ts
@@ -86,15 +85,15 @@ ffill as (
         dim.pool_id,
         dim.collateral_type,
         coalesce(
-            last(debt) over (
+            last(debt.debt) over (
                 partition by dim.collateral_type, dim.pool_id
                 order by dim.ts
                 rows between unbounded preceding and current row
             ),
             0
         ) as debt,
         coalesce(
-            last(collateral_value) over (
+            last(collateral.collateral_value) over (
                 partition by dim.collateral_type, dim.pool_id
                 order by dim.ts
                 rows between unbounded preceding and current row
@@ -141,6 +140,16 @@ hourly_rewards as (
         {{ ref('fct_pool_rewards_hourly_eth_mainnet') }}
 ),
 
+hourly_migration as (
+    select
+        ts,
+        pool_id,
+        collateral_type,
+        hourly_debt_migrated
+    from
+        {{ ref('fct_core_migration_hourly_eth_mainnet') }}
+),
+
 hourly_returns as (
     select
         pnl.ts,
@@ -152,9 +161,16 @@ hourly_returns as (
             iss.hourly_issuance,
             0
         ) as hourly_issuance,
+        coalesce(
+            migration.hourly_debt_migrated,
+            0
+        ) as hourly_debt_migrated,
         pnl.hourly_pnl + coalesce(
             iss.hourly_issuance,
             0
+        ) + coalesce(
+            migration.hourly_debt_migrated,
+            0
         ) as hourly_pnl,
         coalesce(
             rewards.rewards_usd,
@@ -169,19 +185,20 @@ hourly_returns as (
         end as hourly_rewards_pct,
         case
             when pnl.collateral_value = 0 then 0
-            else
-                (coalesce(iss.hourly_issuance, 0) + pnl.hourly_pnl)
-                / pnl.collateral_value
+            else (
+                coalesce(iss.hourly_issuance, 0)
+                + pnl.hourly_pnl
+                + coalesce(migration.hourly_debt_migrated, 0)
+            ) / pnl.collateral_value
         end as hourly_pnl_pct,
         case
             when pnl.collateral_value = 0 then 0
-            else
-                (
-                    coalesce(rewards.rewards_usd, 0)
-                    + pnl.hourly_pnl
-                    + coalesce(iss.hourly_issuance, 0)
-                )
-                / pnl.collateral_value
+            else (
+                coalesce(rewards.rewards_usd, 0)
+                + pnl.hourly_pnl
+                + coalesce(iss.hourly_issuance, 0)
+                + coalesce(migration.hourly_debt_migrated, 0)
+            ) / pnl.collateral_value
         end as hourly_total_pct
     from
         hourly_pnl as pnl
@@ -199,6 +216,15 @@ hourly_returns as (
             ) = lower(
                 iss.collateral_type
             )
+    left join hourly_migration as migration
+        on
+            pnl.ts = migration.ts
+            and pnl.pool_id = migration.pool_id
+            and lower(
+                pnl.collateral_type
+            ) = lower(
+                migration.collateral_type
+            )
 )
 
 select
@@ -209,6 +235,7 @@ select
     debt,
     hourly_issuance,
     hourly_pnl,
+    hourly_debt_migrated,
     rewards_usd,
     hourly_pnl_pct,
     hourly_rewards_pct,

diff --git a/transformers/synthetix/models/raw/base/mainnet/core/core_vault_debt_base_mainnet.sql b/transformers/synthetix/models/raw/base/mainnet/core/core_vault_debt_base_mainnet.sql
@@ -5,7 +5,7 @@ with base as (
         chain_id,
         pool_id,
         collateral_type,
-        cast(
+        CAST(
             value_1 as numeric
         ) as debt
     from
@@ -18,19 +18,24 @@ with base as (
 )
 
 select
-    to_timestamp(blocks.timestamp) as ts,
-    cast(
+    TO_TIMESTAMP(
+        blocks.timestamp
+    ) as ts,
+    CAST(
         blocks.block_number as integer
     ) as block_number,
     base.contract_address,
-    cast(
+    CAST(
         base.pool_id as integer
     ) as pool_id,
-    cast(
+    CAST(
         base.collateral_type as varchar
     ) as collateral_type,
     {{ convert_wei('base.debt') }} as debt
 from
     base
-inner join {{ source('raw_base_mainnet', 'blocks_parquet') }} as blocks
+inner join {{ source(
+        'raw_base_mainnet',
+        'blocks_parquet'
+    ) }} as blocks
     on base.block_number = blocks.block_number
diff --git a/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_collateral_eth_mainnet.sql b/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_collateral_eth_mainnet.sql
@@ -5,10 +5,10 @@ with base as (
         chain_id,
         pool_id,
         collateral_type,
-        CAST(
+        cast(
             amount as numeric
         ) as amount,
-        CAST(
+        cast(
             "value" as numeric
         ) as collateral_value
     from
@@ -21,18 +21,22 @@ with base as (
 )
 
 select
-    blocks.ts,
+    to_timestamp(
+        blocks.timestamp
+    ) as ts,
     base.block_number,
     base.contract_address,
-    CAST(
+    cast(
         base.pool_id as integer
     ) as pool_id,
-    CAST(
+    cast(
         base.collateral_type as varchar
     ) as collateral_type,
     {{ convert_wei('base.amount') }} as amount,
     {{ convert_wei('base.collateral_value') }} as collateral_value
-from
-    base
-inner join {{ source('raw_eth_mainnet', 'blocks_parquet') }} as blocks
+from base
+join {{ source(
+    'raw_eth_mainnet',
+    'blocks_parquet'
+) }} as blocks
     on base.block_number = blocks.block_number
diff --git a/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_debt_eth_mainnet.sql b/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_debt_eth_mainnet.sql
@@ -5,7 +5,7 @@ with base as (
         chain_id,
         pool_id,
         collateral_type,
-        CAST(
+        cast(
             value_1 as numeric
         ) as debt
     from
@@ -18,17 +18,21 @@ with base as (
 )
 
 select
-    blocks.ts,
+    to_timestamp(
+        blocks.timestamp
+    ) as ts,
     base.block_number,
     base.contract_address,
-    CAST(
+    cast(
         base.pool_id as integer
     ) as pool_id,
-    CAST(
+    cast(
         base.collateral_type as varchar
     ) as collateral_type,
     {{ convert_wei('base.debt') }} as debt
-from
-    base
-inner join {{ source('raw_eth_mainnet', 'blocks_parquet') }} as blocks
+from base
+join {{ source(
+    'raw_eth_mainnet',
+    'blocks_parquet'
+) }} as blocks
     on base.block_number = blocks.block_number