Merge branch 'main' of github.com:Layr-Labs/eigenda-proxy into epocia…

…sk--feat-init-v2-scaffolds

README.md

@@ -269,10 +269,9 @@ To quickly set up monitoring dashboard, add eigenda-proxy metrics endpoint to a
 | `--eigenda.signer-private-key-hex`          |                                 | `$EIGENDA_PROXY_EIGENDA_SIGNER_PRIVATE_KEY_HEX`          | Hex-encoded signer private key. This key should not be associated with an Ethereum address holding any funds.                                                                                                 |
 | `--eigenda.status-query-retry-interval`     | `5s`                            | `$EIGENDA_PROXY_EIGENDA_STATUS_QUERY_INTERVAL`           | Interval between retries when awaiting network blob finalization. Default is 5 seconds.                                                                                                                       |
 | `--eigenda.status-query-timeout`            | `30m0s`                         | `$EIGENDA_PROXY_EIGENDA_STATUS_QUERY_TIMEOUT`            | Duration to wait for a blob to finalize after being sent for dispersal. Default is 30 minutes.                                                                                                                |
-| `--log.color`                               | `false`                         | `$EIGENDA_PROXY_LOG_COLOR`                               | Color the log output if in terminal mode.                                                                                                                                                                     |
-| `--log.format`                              | `text`                          | `$EIGENDA_PROXY_LOG_FORMAT`                              | Format the log output. Supported formats: 'text', 'terminal', 'logfmt', 'json', 'json-pretty'.                                                                                                                |
-| `--log.level`                               | `INFO`                          | `$EIGENDA_PROXY_LOG_LEVEL`                               | The lowest log level that will be output.                                                                                                                                                                     |
-| `--log.pid`                                 | `false`                         | `$EIGENDA_PROXY_LOG_PID`                                 | Show pid in the log.                                                                                                                                                                                          |
+| `--log.format`                              | `text`                          | `$EIGENDA_PROXY_LOG_FORMAT`                              | The format of the log file. Accepted options are 'json' and 'text' (default: "json")                                                                                                               |
+| `--log.level`                               | `INFO`                          | `$EIGENDA_PROXY_LOG_LEVEL`                               | The lowest log level that will be output. The lowest log level that will be output. Accepted options are "debug", "info", "warn" "error"                                                                                                                                                                     |
+| `log.path`                                  | `""`                              | `$EIGENDA_PROXY_LOG_PATH`                                | Path to file where logs will be written |
 | `--memstore.enabled`                        | `false`                         | `$EIGENDA_PROXY_MEMSTORE_ENABLED`                        | Whether to use mem-store for DA logic.                                                                                                                                                                        |
 | `--memstore.expiration`                     | `25m0s`                         | `$EIGENDA_PROXY_MEMSTORE_EXPIRATION`                     | Duration that a mem-store blob/commitment pair are allowed to live.                                                                                                                                           |
 | `--memstore.put-latency`                    | `0`                             | `$EIGENDA_PROXY_MEMSTORE_PUT_LATENCY`                    | Artificial latency added for memstore backend to mimic EigenDA's dispersal latency.                                                                                                                           |

cmd/server/entrypoint.go

@@ -5,25 +5,34 @@ import (
 	"encoding/json"
 	"fmt"
 
+	proxy_logging "github.com/Layr-Labs/eigenda-proxy/logging"
+	"github.com/Layr-Labs/eigensdk-go/logging"
+
 	"github.com/Layr-Labs/eigenda-proxy/metrics"
 	"github.com/Layr-Labs/eigenda-proxy/server"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/urfave/cli/v2"
 
 	"github.com/ethereum-optimism/optimism/op-service/ctxinterrupt"
-	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 )
 
 func StartProxySvr(cliCtx *cli.Context) error {
-	log := oplog.NewLogger(oplog.AppOut(cliCtx), oplog.ReadCLIConfig(cliCtx)).New("role", "eigenda_proxy")
-	oplog.SetGlobalLogHandler(log.Handler())
+	logCfg, err := proxy_logging.ReadLoggerCLIConfig(cliCtx)
+	if err != nil {
+		return err
+	}
+
+	log, err := proxy_logging.NewLogger(*logCfg)
+	if err != nil {
+		return err
+	}
+
 	log.Info("Starting EigenDA Proxy Server", "version", Version, "date", Date, "commit", Commit)
 
 	cfg := server.ReadCLIConfig(cliCtx)
 	if err := cfg.Check(); err != nil {
 		return err
 	}
-	err := prettyPrintConfig(cliCtx, log)
+	err = prettyPrintConfig(cliCtx, log)
 	if err != nil {
 		return fmt.Errorf("failed to pretty print config: %w", err)
 	}
@@ -72,7 +81,7 @@ func StartProxySvr(cliCtx *cli.Context) error {
 }
 
 // TODO: we should probably just change EdaClientConfig struct definition in eigenda-client
-func prettyPrintConfig(cliCtx *cli.Context, log log.Logger) error {
+func prettyPrintConfig(cliCtx *cli.Context, log logging.Logger) error {
 	// we read a new config which we modify to hide private info in order to log the rest
 	cfg := server.ReadCLIConfig(cliCtx)
 	if cfg.EigenDAConfig.EdaV1ClientConfig.SignerPrivateKeyHex != "" {

common/common.go

@@ -6,6 +6,12 @@ import (
 	"strings"
 )
 
+const GlobalPrefix = "EIGENDA_PROXY"
+
+func PrefixEnvVar(prefix, suffix string) []string {
+	return []string{prefix + "_" + suffix}
+}
+
 // Helper utility functions //
 
 func ContainsDuplicates[P comparable](s []P) bool {

docker-compose.yaml

@@ -1,22 +1,82 @@
+## The following is a proxy instance 
+## pointed to redis for storage caching and S3
+## for storage failovers
+
 services:
+  ## Used as secondary read failover target
+  minio:
+    image: minio/minio:latest
+    container_name: minio
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    command: server /data
+    volumes:
+      - minio_data:/data
+
+  minio-init:
+    ## Seed test bucket
+    image: minio/mc:latest
+    depends_on:
+      - minio
+    entrypoint: ["/bin/sh", "-c", "/usr/bin/create-bucket.sh"]
+    volumes:
+      - ./scripts/create-test-s3-bucket.sh:/usr/bin/create-bucket.sh
+
+  redis:
+    image: redis:latest
+    container_name: redis
+    command: redis-server --requirepass redispassword
+    environment:
+      - REDIS_PASSWORD=redispassword
+    ports:
+      - "6379:6379"
+
   eigenda_proxy:
+    depends_on:
+      - minio-init
     build:
       context: .
       dockerfile: Dockerfile
     container_name: eigenda-proxy
     environment:
+      - EIGENDA_PROXY_LOG_LEVEL=debug
       - EIGENDA_PROXY_ADDR=0.0.0.0
       - EIGENDA_PROXY_PORT=4242
-      - EIGENDA_PROXY_MEMSTORE_ENABLED=false
+      ## Turn this off to talk to actual eigenda network
+      - EIGENDA_PROXY_MEMSTORE_ENABLED=true
       - EIGENDA_PROXY_MEMSTORE_EXPIRATION=45m
-      - EIGENDA_PROXY_EIGENDA_SIGNER_PRIVATE_KEY_HEX=$PRIVATE_KEY
+      - EIGENDA_PROXY_EIGENDA_CERT_VERIFICATION_DISABLED=true
+      - EIGENDA_PROXY_EIGENDA_SIGNER_PRIVATE_KEY_HEX=${PRIVATE_KEY}
       - EIGENDA_PROXY_EIGENDA_DISPERSER_RPC=disperser-holesky.eigenda.xyz:443
       - EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0xD4A7E1Bd8015057293f0D0A557088c286942e84b
-      - EIGENDA_PROXY_EIGENDA_ETH_RPC=$ETH_RPC
+      - EIGENDA_PROXY_EIGENDA_ETH_RPC=https://ethereum-holesky-rpc.publicnode.com
       - EIGENDA_PROXY_EIGENDA_ETH_CONFIRMATION_DEPTH=0
       - EIGENDA_PROXY_METRICS_ADDR=0.0.0.0
       - EIGENDA_PROXY_METRICS_ENABLED=true
       - EIGENDA_PROXY_METRICS_PORT=7300
+      ## S3
+      - EIGENDA_PROXY_S3_CREDENTIAL_TYPE=static
+      - EIGENDA_PROXY_S3_ACCESS_KEY_ID=minioadmin
+      - EIGENDA_PROXY_S3_ACCESS_KEY_SECRET=minioadmin
+      - EIGENDA_PROXY_S3_BUCKET=eigenda-proxy-test
+      - EIGENDA_PROXY_S3_PATH=""
+      - EIGENDA_PROXY_S3_ENDPOINT=minio:9000
+      - EIGENDA_PROXY_S3_ENABLE_TLS=false
+
+      # Redis Configuration
+      - EIGENDA_PROXY_REDIS_DB=0
+      - EIGENDA_PROXY_REDIS_ENDPOINT=redis:6379
+      - EIGENDA_PROXY_REDIS_PASSWORD=redispassword
+      - EIGENDA_PROXY_REDIS_EVICTION=24h0m0s
+
+      ## Secondary routing
+      - EIGENDA_PROXY_STORAGE_FALLBACK_TARGETS=s3
+      - EIGENDA_PROXY_STORAGE_CACHE_TARGETS=redis
+
     ports:
       - 4242:4242
       - 7300:7300
@@ -44,14 +104,6 @@ services:
     depends_on:
       - prometheus
 
-  traffic-generator:
-    image: alpine:latest
-    build: scripts/
-    container_name: traffic_generator
-    depends_on:
-      - eigenda_proxy
-    volumes:
-      - ./scripts/:/scripts/
-
 volumes:
   grafana-data:
+  minio_data:

e2e/setup.go

@@ -18,14 +18,12 @@ import (
 	"github.com/Layr-Labs/eigenda-proxy/verify/v1"
 	"github.com/Layr-Labs/eigenda/api/clients"
 	"github.com/Layr-Labs/eigenda/encoding/kzg"
+	"github.com/Layr-Labs/eigensdk-go/logging"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"golang.org/x/exp/rand"
 
-	oplog "github.com/ethereum-optimism/optimism/op-service/log"
-	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
-
 	miniotc "github.com/testcontainers/testcontainers-go/modules/minio"
 	redistc "github.com/testcontainers/testcontainers-go/modules/redis"
 )
@@ -243,7 +241,7 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
 	default:
 		cfg = server.CLIConfig{
 			EigenDAConfig: eigendaCfg,
-			MetricsCfg:    opmetrics.CLIConfig{},
+			MetricsCfg:    metrics.CLIConfig{},
 		}
 	}
 
@@ -252,17 +250,13 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
 
 type TestSuite struct {
 	Ctx     context.Context
-	Log     log.Logger
+	Log     logging.Logger
 	Server  *server.Server
 	Metrics *metrics.EmulatedMetricer
 }
 
 func CreateTestSuite(testSuiteCfg server.CLIConfig) (TestSuite, func()) {
-	log := oplog.NewLogger(os.Stdout, oplog.CLIConfig{
-		Level:  log.LevelDebug,
-		Format: oplog.FormatLogFmt,
-		Color:  true,
-	}).New("role", svcName)
+	log := logging.NewTextSLogger(os.Stdout, &logging.SLoggerOptions{})
 
 	m := metrics.NewEmulatedMetricer()
 	ctx := context.Background()

flags/flags.go

@@ -2,20 +2,22 @@ package flags
 
 import (
 	"github.com/Layr-Labs/eigenda-proxy/flags/eigendaflags"
+	"github.com/Layr-Labs/eigenda-proxy/logging"
+	"github.com/Layr-Labs/eigenda-proxy/metrics"
 	"github.com/Layr-Labs/eigenda-proxy/store"
 	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
 	"github.com/Layr-Labs/eigenda-proxy/verify/v1"
 	"github.com/urfave/cli/v2"
 
-	opservice "github.com/ethereum-optimism/optimism/op-service"
-	oplog "github.com/ethereum-optimism/optimism/op-service/log"
-	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	"github.com/Layr-Labs/eigenda-proxy/common"
 )
 
 const (
 	EigenDAClientCategory      = "EigenDA Client"
+	LoggingFlagsCategory       = "Logging"
+	MetricsFlagCategory        = "Metrics"
 	EigenDADeprecatedCategory  = "DEPRECATED EIGENDA CLIENT FLAGS -- THESE WILL BE REMOVED IN V2.0.0"
 	MemstoreFlagsCategory      = "Memstore (for testing purposes - replaces EigenDA backend)"
 	StorageFlagsCategory       = "Storage"
@@ -31,26 +33,20 @@ const (
 	PortFlagName       = "port"
 )
 
-const EnvVarPrefix = "EIGENDA_PROXY"
-
-func prefixEnvVars(name string) []string {
-	return opservice.PrefixEnvVar(EnvVarPrefix, name)
-}
-
 func CLIFlags() []cli.Flag {
 	// TODO: Decompose all flags into constituent parts based on their respective category / usage
 	flags := []cli.Flag{
 		&cli.StringFlag{
 			Name:    ListenAddrFlagName,
 			Usage:   "Server listening address",
 			Value:   "0.0.0.0",
-			EnvVars: prefixEnvVars("ADDR"),
+			EnvVars: common.PrefixEnvVar(common.GlobalPrefix, "ADDR"),
 		},
 		&cli.IntFlag{
 			Name:    PortFlagName,
 			Usage:   "Server listening port",
 			Value:   3100,
-			EnvVars: prefixEnvVars("PORT"),
+			EnvVars: common.PrefixEnvVar(common.GlobalPrefix, "PORT"),
 		},
 	}
 
@@ -62,15 +58,15 @@ var Flags = []cli.Flag{}
 
 func init() {
 	Flags = CLIFlags()
-	Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...)
-	Flags = append(Flags, opmetrics.CLIFlags(EnvVarPrefix)...)
-	Flags = append(Flags, eigendaflags.CLIFlags(EnvVarPrefix, EigenDAClientCategory)...)
-	Flags = append(Flags, eigendaflags.DeprecatedCLIFlags(EnvVarPrefix, EigenDADeprecatedCategory)...)
-	Flags = append(Flags, store.CLIFlags(EnvVarPrefix, StorageFlagsCategory)...)
-	Flags = append(Flags, store.DeprecatedCLIFlags(EnvVarPrefix, StorageDeprecatedCategory)...)
-	Flags = append(Flags, redis.CLIFlags(EnvVarPrefix, RedisCategory)...)
-	Flags = append(Flags, s3.CLIFlags(EnvVarPrefix, S3Category)...)
-	Flags = append(Flags, memstore.CLIFlags(EnvVarPrefix, MemstoreFlagsCategory)...)
-	Flags = append(Flags, verify.CLIFlags(EnvVarPrefix, VerifierCategory)...)
-	Flags = append(Flags, verify.DeprecatedCLIFlags(EnvVarPrefix, VerifierDeprecatedCategory)...)
+	Flags = append(Flags, logging.CLIFlags(common.GlobalPrefix, LoggingFlagsCategory)...)
+	Flags = append(Flags, metrics.CLIFlags(common.GlobalPrefix, MetricsFlagCategory)...)
+	Flags = append(Flags, eigendaflags.CLIFlags(common.GlobalPrefix, EigenDAClientCategory)...)
+	Flags = append(Flags, eigendaflags.DeprecatedCLIFlags(common.GlobalPrefix, EigenDADeprecatedCategory)...)
+	Flags = append(Flags, store.CLIFlags(common.GlobalPrefix, StorageFlagsCategory)...)
+	Flags = append(Flags, store.DeprecatedCLIFlags(common.GlobalPrefix, StorageDeprecatedCategory)...)
+	Flags = append(Flags, redis.CLIFlags(common.GlobalPrefix, RedisCategory)...)
+	Flags = append(Flags, s3.CLIFlags(common.GlobalPrefix, S3Category)...)
+	Flags = append(Flags, memstore.CLIFlags(common.GlobalPrefix, MemstoreFlagsCategory)...)
+	Flags = append(Flags, verify.CLIFlags(common.GlobalPrefix, VerifierCategory)...)
+	Flags = append(Flags, verify.DeprecatedCLIFlags(common.GlobalPrefix, VerifierDeprecatedCategory)...)
 }

go.mod

@@ -7,6 +7,7 @@ toolchain go1.22.7
 require (
 	github.com/Layr-Labs/eigenda v0.8.6-rc.0.0.20250201021742-a13af0245831
 	github.com/Layr-Labs/eigenda-proxy/client v0.0.0-00010101000000-000000000000
+	github.com/Layr-Labs/eigensdk-go v0.2.0-beta.1.0.20250118004418-2a25f31b3b28
 	github.com/avast/retry-go/v4 v4.6.0
 	github.com/consensys/gnark-crypto v0.12.1
 	github.com/ethereum-optimism/optimism v1.9.5
@@ -223,7 +224,6 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
-	github.com/pingcap/errors v0.11.4 // indirect
 	github.com/pion/datachannel v1.5.8 // indirect
 	github.com/pion/dtls/v2 v2.2.12 // indirect
 	github.com/pion/ice/v2 v2.3.34 // indirect

go.sum

@@ -24,10 +24,6 @@ github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR
 github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/Layr-Labs/cerberus-api v0.0.2-0.20250117193600-e69c5e8b08fd h1:prMzW4BY6KZtWEanf5EIsyHzIZKCNV2mVIXrE6glRRM=
 github.com/Layr-Labs/cerberus-api v0.0.2-0.20250117193600-e69c5e8b08fd/go.mod h1:Lm4fhzy0S3P7GjerzuseGaBFVczsIKmEhIjcT52Hluo=
-github.com/Layr-Labs/eigenda v0.8.6-rc.0.0.20250129221107-aed77359f08e h1:OVGnKM417dJSQLMKDA/S2itGf9O5QZhD+Or4zk2shpM=
-github.com/Layr-Labs/eigenda v0.8.6-rc.0.0.20250129221107-aed77359f08e/go.mod h1:hdvXBB3dggZnM0xDOnfh9FIUXlmHekPu1Kf7IwdtuUU=
-github.com/Layr-Labs/eigenda v0.8.6-rc.0.0.20250130173836-0d293cc03198 h1:mZ6bEyJmE21cBUR6EA+PD3X8kx4ZMIrRauTN9OzVnho=
-github.com/Layr-Labs/eigenda v0.8.6-rc.0.0.20250130173836-0d293cc03198/go.mod h1:hdvXBB3dggZnM0xDOnfh9FIUXlmHekPu1Kf7IwdtuUU=
 github.com/Layr-Labs/eigenda v0.8.6-rc.0.0.20250201021742-a13af0245831 h1:hfH2DruVKRVEksM0cX6bHwvuNncB6iLXqTjUH3A4CcM=
 github.com/Layr-Labs/eigenda v0.8.6-rc.0.0.20250201021742-a13af0245831/go.mod h1:MO3EyBXCmhzttrmmgqmIkzDg4gtqWjU+0fxSN0Q1EmM=
 github.com/Layr-Labs/eigensdk-go v0.2.0-beta.1.0.20250118004418-2a25f31b3b28 h1:Wig5FBBizIB5Z/ZcXJlm7KdOLnrXc6E3DjO63uWRzQM=