feat: Nostra LP Pricer (#214)

* feat: deployed mock pool addresses

* feat: configuration + README

* feat: first draft

* feat: reorganization

* feat: docs

* feat: pool deployments + type fixing

* feat: push onchain + register pools if unregistered

* fix: constants

* feat: print -> logger + error management

* feat: error management + little tests

* feat: refactor

* feat-Nostra-LP-price: Finished! Need Typecheck + Tests

* feat-Nostra-LP-price: Redis test

* feat-Nostra-LP-price: delay

* fix: typecheck + format

* fix: typecheck

* feat-Nostra-LP-price: linters

* added ci

* added ci

* feat-Nostra-LP-price:

* feat-Nostra-LP-price: Clauded

* feat-Nostra-LP-price:

* feat-Nostra-LP-price: Fixed storage bug + test

* feat-Nostra-LP-price:

* feat-Nostra-LP-price: QoL changes

* feat-Nostra-LP-price: Don't crash the whole app on failure

* feat-Nostra-LP-price: Removed logs - poluates

* feat-Nostra-LP-price: Various fixes :)

* feat-Nostra-LP-price: Fixed currency name

* feat-Nostra-LP-price: less pools

* feat-Nostra-LP-price: Update abis

* feat-Nostra-LP-price: Review please 🙏

* feat-Nostra-LP-price: Re-added USDC/USDT pool

* feat-Nostra-LP-price: Fixed LP computation

* feat-Nostra-LP-price: Fixed python bug

* feat-Nostra-LP-price: Fixed forked client

* feat-Nostra-LP-price: Increased checkpointer wait time

* feat-Nostra-LP-price: Big increment :)

* feat-Nostra-LP-price: oops

* feat-Nostra-LP-price: Final stuff

* feat-Nostra-LP-price: Updated oracle to main + Removed LpPricer

* feat-Nostra-LP-price: Referenced lp-pricer to the monorepo

* feat-Nostra-LP-price: CI Fix

* feat-Nostra-LP-price: Fighting with CI

* feat-Nostra-LP-price: Unit test config

* feat-Nostra-LP-price: Should be gud

* feat-Nostra-LP-price: Data pruning + test

* feat-Nostra-LP-price: Removed unused/useless function

* feat-Nostra-LP-price: More robust checks at startup for Pricer

* feat-Nostra-LP-price: Logs happen in their own function

* feat-Nostra-LP-price: Fixed the duplicated logs / formatting differences

* feat-Nostra-LP-price: Removed new asset

* feat-Nostra-LP-price: Fixes from review

* fix: other fixes

* feat-Nostra-LP-price:

* feat-Nostra-LP-price: More precision for computations

---------

Co-authored-by: akhercha <[email protected]>
Co-authored-by: devops72-sre <>
Co-authored-by: 0xevolve <[email protected]>

.github/workflows/linters.yml

@@ -22,6 +22,7 @@ jobs:
             vrf-listener,
             checkpointer,
             merkle-maker,
+            lp-pricer,
           ]
       fail-fast: false
     steps:

.github/workflows/tests.yml

@@ -22,6 +22,7 @@ jobs:
             vrf-listener,
             checkpointer,
             merkle-maker,
+            lp-pricer,
           ]
       fail-fast: false
     steps:
@@ -44,26 +45,26 @@ jobs:
         id: cache-contracts
         uses: actions/cache@v3
         with:
-          path: ./pragma-oracle/target
-          key: ${{ runner.os }}-contracts-${{ hashFiles('./pragma-oracle/src', 'poetry.lock') }}
+          path: ./pragma-oracle/pragma-oracle/target
+          key: ${{ runner.os }}-contracts-${{ hashFiles('./pragma-oracle/pragma-oracle/src', 'poetry.lock') }}
 
       - uses: software-mansion/setup-scarb@v1
         with:
           tool-versions: pragma-oracle/.tool-versions
 
       - name: Compile contracts
         if: steps.cache-contracts.outputs.cache-hit != 'true'
-        working-directory: ./pragma-oracle
+        working-directory: ./pragma-oracle/pragma-oracle
         run: |
           scarb -V
           scarb build
 
       - name: Check ABIs are up-to-date
         run: |
           # Find and compare *.sierra.json files in both directories
-          for file in $(find ./pragma-oracle/target/dev -type f -name "*.sierra.json"); do
+          for file in $(find ./pragma-oracle/pragma-oracle/target/dev -type f -name "*.sierra.json"); do
             # Extract the relative path of the file for comparison
-            rel_path=${file#./pragma-oracle/target/dev}
+            rel_path=${file#./pragma-oracle/pragma-oracle/target/dev}
             # Use diff to compare the file with its counterpart in the SDK directory
             diff "$file" "pragma-sdk/pragma_sdk/onchain/abis$rel_path" || (echo "Error: $rel_path in pragma-oracle/target does not match with pragma-sdk/pragma_sdk/onchain/abis/" && exit 1) 
           done

checkpointer/tests/integration/checkpointer_test.py

@@ -88,7 +88,7 @@ async def test_checkpointer_spot(
         private_key=private_key,
     )
 
-    await asyncio.sleep(5)
+    await asyncio.sleep(60)
 
     latest_checkpoint = await pragma_client.get_latest_checkpoint(
         pair_id="BTC/USD",
@@ -238,7 +238,7 @@ async def test_checkpointer_spot_and_future(
         private_key=private_key,
     )
 
-    await asyncio.sleep(10)
+    await asyncio.sleep(60)
 
     latest_checkpoint = await pragma_client.get_latest_checkpoint(
         pair_id="BTC/USD",

checkpointer/tests/integration/constants.py

@@ -24,7 +24,7 @@ def find_repo_root(start_directory: Path) -> Path:
 repo_root = find_repo_root(current_file_directory).parent
 
 SUBMODULE_DIR = repo_root / "pragma-oracle"
-CONTRACTS_COMPILED_DIR = SUBMODULE_DIR / "target/dev"
+CONTRACTS_COMPILED_DIR = SUBMODULE_DIR / "pragma-oracle" / "target" / "dev"
 DEPLOYMENTS_DIR = SUBMODULE_DIR / "deployments"
 
 

infra/nostra-lp-pricer/Dockerfile

@@ -0,0 +1,41 @@
+FROM python:3.12-slim AS base
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    POETRY_VERSION=1.8.3  \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    POETRY_NO_INTERACTION=1
+
+ENV PATH="/root/.local/bin:${PATH}"
+
+FROM base as builder
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y \
+    gcc \
+    libgmp3-dev \
+    pipx
+
+RUN apt-get update && apt-get install -y bash curl && curl -1sLf \
+'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.deb.sh' | bash \
+&& apt-get update && apt-get install -y infisical
+
+
+RUN pipx install poetry
+COPY pragma-sdk/ /opt/pragma-sdk/
+COPY pragma-utils/ /opt/pragma-utils/
+COPY lp-pricer/ /opt/lp-pricer/
+WORKDIR /opt/lp-pricer
+RUN poetry env use 3.12
+RUN poetry install
+
+FROM base as final
+COPY --from=builder /usr/bin/infisical /usr/bin/infisical
+COPY --from=builder /opt /opt
+COPY infra/nostra-lp-pricer/entrypoint.sh /opt/lp-pricer/
+COPY infra/nostra-lp-pricer/config/ /opt/lp-pricer/config/
+WORKDIR /opt/lp-pricer
+ENTRYPOINT ["bash","/opt/lp-pricer/entrypoint.sh"]

infra/nostra-lp-pricer/buildspec.yml

@@ -0,0 +1,30 @@
+version: 0.2
+phases:
+  pre_build:
+    commands:
+      - echo Logging in to Amazon ECR...
+      - aws --version
+      - aws ecr get-login-password --region $ECR_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com
+      - REPOSITORY_URI=$AWS_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/$ECR_REPOSITORY_NAME
+      - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7)
+      - IMAGE_TAG=${COMMIT_HASH:=latest}
+  build:
+    commands:
+      - echo Build started on `date`
+      - echo Building the Docker image...
+      - ls -ltr
+      - docker build -f infra/nostra-lp-pricer/Dockerfile -t $REPOSITORY_URI:latest .
+      - docker tag $REPOSITORY_URI:latest $REPOSITORY_URI:$IMAGE_TAG
+  post_build:
+    commands:
+      - echo Build completed on `date`
+      - echo Pushing the Docker images...
+      - docker push $REPOSITORY_URI:latest
+      - docker push $REPOSITORY_URI:$IMAGE_TAG
+      - echo Writing image definitions file...
+      - printf '[{"name":"%s","imageUri":"%s"}]' $ECS_CONTAINER_NAME $REPOSITORY_URI:$IMAGE_TAG > imagedefinitions.json
+artifacts:
+  files:
+    - imagedefinitions.json
+    - infra/nostra-lp-pricer/config.yml
+  discard-paths: yes

infra/nostra-lp-pricer/config.yml

@@ -0,0 +1,9 @@
+path: "/"
+container_port: 8080
+health_check_path: "/"
+container_environment:
+  - region: "eu-west-3"
+  - prefix: "/conf/{{ SERVICE_NAME }}/{{ RUN_ENV }}"
+  - keys:
+      - INFISICAL_ENV
+      - INFISICAL_APP_PATH

infra/nostra-lp-pricer/config/config.mainnet.yaml

@@ -0,0 +1,8 @@
+pool_addresses:
+  - '0x00c318445d5a5096e2ad086452d5c97f65a9d28cafe343345e0fa70da0841295' # USDC/USDT - Degen
+  # - '0x0362ec0c49a9c8f2d322d0ba6a8ec1214b9e4f7e80a17d462ec2585362547d95' # USDC/DAI - Degen
+  - '0x01a2de9f2895ac4e6cb80c11ecc07ce8062a4ae883f64cb2b1dc6724b85e897d' # STRK/ETH - Degen
+  - '0x042543c7d220465bd3f8f42314b51f4f3a61d58de3770523b281da61dbf27c8a' # STRK/USDC - Degen
+  - '0x05e03162008d76cf645fe53c6c13a7a5fce745e8991c6ffe94400d60e44c210a' # ETH/USDC - Degen
+  - '0x01583919ffd78e87fa28fdf6b6a805fe3ddf52f754a63721dcd4c258211129a6' # WBTC/ETH - Degen
+  # - '0x076def79cc9a3a375779c163ad12996f99fbeb4acd68d7041529159bde897160' # nstSTRK/STRK - Degen

infra/nostra-lp-pricer/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -euo pipefail
+export INFISICAL_TOKEN=$(infisical login --method=universal-auth --client-id=${INFISICAL_CLIENT_ID} --client-secret=${INFISICAL_CLIENT_SECRET} --silent --plain)
+infisical export  --projectId=${INFISICAL_PROJECT_ID} --env=${INFISICAL_ENV}  --path=${INFISICAL_APP_PATH} > .env
+source .env
+exec /opt/lp-pricer/.venv/bin/python3.12 lp_pricer/main.py ${ARGS}

lp-pricer/README.md

@@ -0,0 +1,52 @@
+# Lp Pricer
+
+Service used to price Defi Pools using the `LpFetcher` from the SDK.
+
+### Usage
+
+The service is ran through the CLI, to have more information you can use the `--help` command:
+
+```bash
+.venv ❯ poetry run lp_pricer --help
+
+Usage: lp_pricer [OPTIONS]
+
+  Lp Pricer entry point.
+
+Options:
+  --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL]
+                                  Logging level.
+
+  -c, --config-file PATH          Path to YAML configuration file.  [required]
+
+  -n, --network [sepolia|mainnet|devnet|pragma_devnet]
+                                  On which networks the checkpoints will be
+                                  set.  [required]
+
+  --redis-host TEXT               Host where the Redis service is live. Format
+                                  is HOST:PORT, example: localhost:6379
+
+  --rpc-url TEXT                  RPC url used by the onchain client.
+
+  --publisher-name TEXT           Name of the publisher of the LP Pricer.
+                                  [required]
+
+  --publisher-address TEXT        Address of the publisher of the LP Pricer.
+                                  [required]
+
+  -p, --private-key TEXT          Private key of the publisher. Format:
+                                  aws:secret_name, plain:private_key,
+                                  env:ENV_VAR_NAME, or
+                                  keystore:PATH/TO/THE/KEYSTORE:PASSWORD
+                                  [required]
+
+  --help                          Show this message and exit.
+```
+
+For example:
+
+```sh
+poetry run lp_pricer -c ./config/config.example.yaml --publisher-name $PUBLISHER_NAME --publisher-address $PUBLISHER_ADDRESS -p plain:$PUBLISHER_PV_KEY
+```
+
+Will start storing reserves & supply for the provided pools until there is enough to price them and will push the prices on chain.

lp-pricer/VERSION

@@ -0,0 +1 @@
+2.2.0

lp-pricer/config/config.example.yaml

@@ -0,0 +1,3 @@
+pool_addresses:
+  - '0x00c318445d5a5096e2ad086452d5c97f65a9d28cafe343345e0fa70da0841295'
+  - '0x00da114221cb83fa859dbdb4c44beeaa0bb37c7537ad5ae66fe5e0efd20e6eb3'

lp-pricer/lp_pricer/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.2.0"

lp-pricer/lp_pricer/configs/pools_config.py

@@ -0,0 +1,18 @@
+import yaml
+
+from typing import List
+
+from pydantic import BaseModel
+
+
+class PoolsConfig(BaseModel):
+    pool_addresses: List[str]
+
+    @classmethod
+    def from_yaml(cls, path: str) -> "PoolsConfig":
+        with open(path, "r") as file:
+            pools_config = yaml.safe_load(file)
+        return cls(**pools_config)
+
+    def get_all_pools(self) -> List[int]:
+        return [int(address, 16) for address in self.pool_addresses]