fix: improve deposits query

.env.test

@@ -0,0 +1,8 @@
+DATABASE_HOST=postgres
+DATABASE_PORT=5432
+DATABASE_USER=user
+DATABASE_PASSWORD=password
+DATABASE_NAME=indexer-test-database
+
+REDIS_HOST=redis
+REDIS_PORT=6379

.github/workflows/ci.yml

@@ -26,10 +26,8 @@ jobs:
           cache: 'pnpm'
       - name: Install dependencies
         run: pnpm install
-      - name: Build
-        run: pnpm build
-      - name: Test
-        run: pnpm test
+      - name: Run tests using docker
+        run: pnpm test:e2e:docker
   build:
     name: Build
     timeout-minutes: 15

.husky/pre-commit

@@ -1 +1 @@
-pnpm turbo run check test
+pnpm turbo run check

docker-compose.e2e.yml

@@ -0,0 +1,43 @@
+services:
+    # Indexer Service
+  indexer-scraper-e2e:
+    build:
+      context: .
+    container_name: indexer_service_e2e
+    volumes:
+      - .:/usr/src/app
+      - indexer-e2e-node-modules:/usr/src/app/node_modules
+    command: [sh, -c, "pnpm build && pnpm db:indexer-database:migrate:run && pnpm test"]
+    env_file:
+      - .env.test
+    tty: true
+    depends_on:
+      - postgres
+      - redis
+
+  # PostgreSQL Service
+  postgres:
+    image: postgres:13-alpine
+    container_name: postgres_e2e
+    env_file:
+      - .env.test
+    environment:
+      TZ: GMT
+      POSTGRES_DB: ${DATABASE_NAME}
+      POSTGRES_USER: ${DATABASE_USER}
+      POSTGRES_PASSWORD: ${DATABASE_PASSWORD}
+    volumes:
+      - indexer-e2e-pgdata:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+
+  # Redis Service
+  redis:
+    image: redis:6-alpine
+    container_name: redis_cache_e2e
+    volumes:
+      - indexer-e2e-redis-volume:/data
+volumes:
+  indexer-e2e-node-modules:
+  indexer-e2e-pgdata:
+  indexer-e2e-redis-volume:

package.json

@@ -7,6 +7,8 @@
     "fix": "turbo fix",
     "check": "turbo check",
     "test": "turbo test",
+    "test:e2e:docker": "docker compose --env-file .env.test -f docker-compose.e2e.yml up --abort-on-container-exit --exit-code-from indexer-scraper-e2e",
+    "test:e2e:docker:prune": "docker compose --env-file .env.test -f docker-compose.e2e.yml down -v",
     "start:indexer": "APP=indexer pnpm --filter @repo/node-app start",
     "start:indexer:prod": "APP=indexer pnpm --filter @repo/node-app start:prod",
     "start:indexer-api": "APP=indexer-api pnpm --filter @repo/node-app start",
@@ -25,7 +27,7 @@
     "turbo": "^2.0.10",
     "husky": "^9.1.4"
   },
-  "packageManager": "pnpm@8.15.6",
+  "packageManager": "pnpm@9.15.5",
   "engines": {
     "node": ">=18"
   }

packages/error-handling/src/utils/assert.ts

@@ -8,7 +8,10 @@ import { AssertError } from "../errors/AssertError";
  * @returns An assertion of `value`.
  * @throws {@link AssertError} if assert's validity fails
  */
-export function assert(value: unknown, message: string): asserts value {
+export function assert(
+  value: unknown,
+  message: string,
+): asserts value is NonNullable<unknown> {
   try {
     return assertModule.ok(value, message);
   } catch (e: unknown) {

packages/indexer-api/.mocharc.json

@@ -4,7 +4,7 @@
   ],
   "spec": "**/*.test.ts",
   "require": [
-    "ts-node/register"
+    "ts-node/register","dotenv/config"
   ],
   "recursive": true
 }

packages/indexer-api/package.json

@@ -14,7 +14,7 @@
     "lint": "eslint --fix",
     "lint:check": "eslint",
     "check": "pnpm format:check && pnpm lint:check && pnpm build:check",
-    "test": "mocha",
+    "test": "DOTENV_CONFIG_PATH=./../../.env.test mocha",
     "coverage": "nyc mocha",
     "test:watch": "mocha --watch"
   },
@@ -24,12 +24,13 @@
   "dependencies": {
     "@repo/error-handling": "workspace:*",
     "@repo/indexer": "workspace:*",
-    "@repo/webhooks": "workspace:*",
     "@repo/indexer-database": "workspace:*",
+    "@repo/webhooks": "workspace:*",
     "@types/express": "^4.17.21",
     "@types/supertest": "^6.0.2",
     "body-parser": "^1.20.2",
     "cors": "^2.8.5",
+    "dotenv": "^16.4.5",
     "ejs": "^3.1.10",
     "express": "^4.19.2",
     "ioredis": "^5.4.1",

packages/indexer-api/src/controllers/deposits.ts

@@ -1,7 +1,11 @@
 import { Request, Response, NextFunction } from "express";
 import * as s from "superstruct";
 import { DepositsService } from "../services/deposits";
-import { DepositsParams, DepositParams } from "../dtos/deposits.dto";
+import {
+  DepositsParams,
+  DepositParams,
+  FilterDepositsParams,
+} from "../dtos/deposits.dto";
 
 export class DepositsController {
   constructor(private service: DepositsService) {}
@@ -33,4 +37,31 @@ export class DepositsController {
       next(err);
     }
   };
+
+  public getUnfilledDeposits = async (
+    req: Request,
+    res: Response,
+    next: NextFunction,
+  ) => {
+    try {
+      const params = s.create(req.query, FilterDepositsParams);
+      const result = await this.service.getUnfilledDeposits(params);
+      return res.json(result);
+    } catch (err) {
+      next(err);
+    }
+  };
+  public getFilledDeposits = async (
+    req: Request,
+    res: Response,
+    next: NextFunction,
+  ) => {
+    try {
+      const params = s.create(req.query, FilterDepositsParams);
+      const result = await this.service.getFilledDeposits(params);
+      return res.json(result);
+    } catch (err) {
+      next(err);
+    }
+  };
 }

packages/indexer-api/src/controllers/fills.ts

@@ -0,0 +1,22 @@
+import { Request, Response, NextFunction } from "express";
+import * as s from "superstruct";
+import { FillsService } from "../services/fills";
+import { UnmatchedFillsParams } from "../dtos/fills.dto";
+
+export class FillsController {
+  constructor(private service: FillsService) {}
+
+  public getUnmatchedFills = async (
+    req: Request,
+    res: Response,
+    next: NextFunction,
+  ) => {
+    try {
+      const params = s.create(req.query, UnmatchedFillsParams);
+      const result = await this.service.getUnmatchedFills(params);
+      return res.json(result);
+    } catch (err) {
+      next(err);
+    }
+  };
+}

packages/indexer-api/src/dtos/deposits.dto.ts

@@ -21,13 +21,17 @@ export const DepositsParams = s.object({
   // some kind of pagination options, skip could be the start point
   skip: s.optional(stringToInt),
   // pagination limit, how many to return after the start, note we convert string to number
-  limit: s.defaulted(stringToInt, 50),
+  limit: s.refine(
+    s.defaulted(stringToInt, 50),
+    "maxLimit",
+    (value) => value <= 1000 || "Limit must not exceed 1000",
+  ),
 });
 
 export type DepositsParams = s.Infer<typeof DepositsParams>;
 
 export const DepositParams = s.object({
-  depositId: s.optional(stringToInt),
+  depositId: s.optional(s.string()),
   originChainId: s.optional(stringToInt),
   depositTxHash: s.optional(s.string()),
   relayDataHash: s.optional(s.string()),
@@ -39,3 +43,74 @@ export const DepositParams = s.object({
 });
 
 export type DepositParams = s.Infer<typeof DepositParams>;
+
+export const FilterDepositsParams = s.object({
+  originChainId: s.optional(stringToInt),
+  destinationChainId: s.optional(stringToInt),
+  startTimestamp: s.optional(stringToInt),
+  endTimestamp: s.optional(stringToInt),
+  skip: s.defaulted(stringToInt, 0),
+  limit: s.refine(
+    s.defaulted(stringToInt, 50),
+    "maxLimit",
+    (value) => value <= 1000 || "Limit must not exceed 1000",
+  ),
+  minSecondsToFill: s.optional(stringToInt),
+});
+
+export type FilterDepositsParams = s.Infer<typeof FilterDepositsParams>;
+
+export type DepositReturnType = {
+  // Fields from V3FundsDeposited
+  id: number;
+  relayHash: string;
+  depositId: string;
+  originChainId: number;
+  destinationChainId: number;
+  depositor: string;
+  recipient: string;
+  inputToken: string;
+  inputAmount: string;
+  outputToken: string;
+  outputAmount: string;
+  message: string;
+  messageHash?: string;
+  exclusiveRelayer: string;
+  exclusivityDeadline?: Date;
+  fillDeadline: Date;
+  quoteTimestamp: Date;
+
+  depositTransactionHash: string;
+  depositBlockNumber: number;
+  depositBlockTimestamp?: Date;
+
+  // Fields from RelayHashInfo
+  status: entities.RelayStatus;
+  depositRefundTxHash?: string;
+  swapTokenPriceUsd?: string;
+  swapFeeUsd?: string;
+  bridgeFeeUsd?: string;
+  inputPriceUsd?: string;
+  outputPriceUsd?: string;
+  fillGasFee?: string;
+  fillGasFeeUsd?: string;
+  fillGasTokenPriceUsd?: string;
+
+  // from fill
+  relayer?: string;
+  fillBlockTimestamp?: Date;
+  fillTransactionHash?: string;
+
+  // from swap
+  swapTransactionHash?: string;
+  swapToken?: string;
+  swapTokenAmount?: string;
+
+  speedups?: {
+    transactionHash: string;
+    updatedRecipient: string;
+    updatedOutputAmount: string;
+    updatedMessage: string;
+    blockNumber: number;
+  }[];
+};

packages/indexer-api/src/dtos/fills.dto.ts

@@ -0,0 +1,25 @@
+import * as s from "superstruct";
+
+const stringToInt = s.coerce(s.number(), s.string(), (value) => {
+  // Ensure the value is a valid integer string
+  if (!/^-?\d+$/.test(value)) {
+    return value;
+  }
+  return parseInt(value, 10);
+});
+
+export const UnmatchedFillsParams = s.object({
+  originChainId: s.optional(stringToInt),
+  destinationChainId: s.optional(stringToInt),
+  startTimestamp: s.optional(stringToInt),
+  endTimestamp: s.optional(stringToInt),
+  relayer: s.optional(s.string()),
+  skip: s.defaulted(stringToInt, 0),
+  limit: s.refine(
+    s.defaulted(stringToInt, 50),
+    "maxLimit",
+    (value) => value <= 1000 || "Limit must not exceed 1000",
+  ),
+});
+
+export type UnmatchedFillsParams = s.Infer<typeof UnmatchedFillsParams>;