indexer: DB pool config & metrics (MystenLabs#13220)

## Description 
- add configurable DB pool size
- monitor DB pool, the total size and idle conn

## Test Plan 

local run and saw metrics on localhost
```
indexer_idle_db_conn{indexer_fullnode="ord-mnt-rpcbig-06.mainnet.sui"} 100
indexer_db_conn_pool_size{indexer_fullnode="ord-mnt-rpcbig-06.mainnet.sui"} 100
```
and logs
```
2023-07-29T20:50:37.370315Z  INFO sui_indexer: DB connection pool size: 100, with idle conn: 100.
```

---
If your changes are not user-facing and not a breaking change, you can
skip the following section. Otherwise, please indicate what changed, and
then add to the Release Notes section as highlighted during the release
process.

### Type of Change (Check all that apply)

- [ ] protocol change
- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes

crates/sui-indexer/src/lib.rs

@@ -318,7 +318,7 @@ struct PgConectionPoolConfig {
 }
 
 impl PgConectionPoolConfig {
-    const DEFAULT_POOL_SIZE: u32 = 10;
+    const DEFAULT_POOL_SIZE: u32 = 100;
     const DEFAULT_CONNECTION_TIMEOUT: Duration = Duration::from_secs(30);
     const DEFAULT_STATEMENT_TIMEOUT: Duration = Duration::from_secs(5);
 
@@ -331,8 +331,12 @@ impl PgConectionPoolConfig {
 
 impl Default for PgConectionPoolConfig {
     fn default() -> Self {
+        let db_pool_size = std::env::var("DB_POOL_SIZE")
+            .ok()
+            .and_then(|s| s.parse::<u32>().ok())
+            .unwrap_or(Self::DEFAULT_POOL_SIZE);
         Self {
-            pool_size: Self::DEFAULT_POOL_SIZE,
+            pool_size: db_pool_size,
             connection_timeout: Self::DEFAULT_CONNECTION_TIMEOUT,
             statement_timeout: Self::DEFAULT_STATEMENT_TIMEOUT,
         }
@@ -342,7 +346,6 @@ impl Default for PgConectionPoolConfig {
 #[derive(Debug, Clone, Copy)]
 struct PgConnectionConfig {
     statement_timeout: Duration,
-    // read_only: bool,
 }
 
 impl diesel::r2d2::CustomizeConnection<PgConnection, diesel::r2d2::Error> for PgConnectionConfig {
@@ -355,13 +358,6 @@ impl diesel::r2d2::CustomizeConnection<PgConnection, diesel::r2d2::Error> for Pg
         ))
         .execute(conn)
         .map_err(diesel::r2d2::Error::QueryError)?;
-
-        // if self.read_only {
-        //     sql_query("SET default_transaction_read_only = 't'")
-        //         .execute(conn)
-        //         .map_err(r2d2::Error::QueryError)?;
-        // }
-
         Ok(())
     }
 }

crates/sui-indexer/src/main.rs

@@ -45,6 +45,26 @@ async fn main() -> Result<(), IndexerError> {
         );
         e
     })?;
+
+    let report_cp = blocking_cp.clone();
+    let report_metrics = indexer_metrics.clone();
+    tokio::spawn(async move {
+        loop {
+            let cp_state = report_cp.state();
+            info!(
+                "DB connection pool size: {}, with idle conn: {}.",
+                cp_state.connections, cp_state.idle_connections
+            );
+            report_metrics
+                .db_conn_pool_size
+                .set(cp_state.connections as i64);
+            report_metrics
+                .idle_db_conn
+                .set(cp_state.idle_connections as i64);
+            tokio::time::sleep(tokio::time::Duration::from_secs(60)).await;
+        }
+    });
+
     if indexer_config.reset_db {
         let mut conn = get_pg_pool_connection(&blocking_cp).map_err(|e| {
             error!(

crates/sui-indexer/src/metrics.rs

@@ -63,6 +63,9 @@ pub struct IndexerMetrics {
     pub get_dynamic_fields_latency: Histogram,
     pub get_dynamic_field_object_latency: Histogram,
     pub get_protocol_config_latency: Histogram,
+    // indexer state metrics
+    pub db_conn_pool_size: IntGauge,
+    pub idle_db_conn: IntGauge,
 }
 
 impl IndexerMetrics {
@@ -352,6 +355,16 @@ impl IndexerMetrics {
                 registry
             )
             .unwrap(),
+            db_conn_pool_size: register_int_gauge_with_registry!(
+                "db_conn_pool_size",
+                "Size of the database connection pool",
+                registry
+            ).unwrap(),
+            idle_db_conn: register_int_gauge_with_registry!(
+                "idle_db_conn",
+                "Number of idle database connections",
+                registry
+            ).unwrap(),
         }
     }
 }