Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: clean up flags and configs #146

Merged
merged 18 commits into from
Sep 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ LDFLAGSSTRING +=-X main.Date=$(BUILD_TIME)
LDFLAGSSTRING +=-X main.Version=$(GIT_TAG)
LDFLAGS := -ldflags "$(LDFLAGSSTRING)"

E2ETEST = INTEGRATION=true go test -timeout 1m -v ./e2e -parallel 4 -deploy-config ../.devnet/devnetL1.json
HOLESKYTEST = TESTNET=true go test -timeout 50m -v ./e2e -parallel 4 -deploy-config ../.devnet/devnetL1.json
E2ETEST = INTEGRATION=true go test -timeout 1m ./e2e -parallel 4 -deploy-config ../.devnet/devnetL1.json
HOLESKYTEST = TESTNET=true go test -timeout 50m ./e2e -parallel 4 -deploy-config ../.devnet/devnetL1.json

.PHONY: eigenda-proxy
eigenda-proxy:
Expand Down
6 changes: 3 additions & 3 deletions cmd/server/entrypoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"

"github.com/Layr-Labs/eigenda-proxy/flags"
"github.com/Layr-Labs/eigenda-proxy/metrics"
"github.com/Layr-Labs/eigenda-proxy/server"
"github.com/urfave/cli/v2"
Expand All @@ -24,15 +25,14 @@ func StartProxySvr(cliCtx *cli.Context) error {
ctx, ctxCancel := context.WithCancel(cliCtx.Context)
defer ctxCancel()

m := metrics.NewMetrics("default")

log.Info("Initializing EigenDA proxy server...")

daRouter, err := server.LoadStoreRouter(ctx, cfg, log)
if err != nil {
return fmt.Errorf("failed to create store: %w", err)
}
server := server.NewServer(cliCtx.String(server.ListenAddrFlagName), cliCtx.Int(server.PortFlagName), daRouter, log, m)
m := metrics.NewMetrics("default")
server := server.NewServer(cliCtx.String(flags.ListenAddrFlagName), cliCtx.Int(flags.PortFlagName), daRouter, log, m)

if err := server.Start(); err != nil {
return fmt.Errorf("failed to start the DA server: %w", err)
Expand Down
4 changes: 2 additions & 2 deletions cmd/server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import (
"github.com/joho/godotenv"
"github.com/urfave/cli/v2"

"github.com/Layr-Labs/eigenda-proxy/flags"
"github.com/Layr-Labs/eigenda-proxy/metrics"
"github.com/Layr-Labs/eigenda-proxy/server"
"github.com/ethereum-optimism/optimism/op-service/cliapp"
oplog "github.com/ethereum-optimism/optimism/op-service/log"
"github.com/ethereum-optimism/optimism/op-service/metrics/doc"
Expand All @@ -26,7 +26,7 @@ func main() {
oplog.SetupDefaults()

app := cli.NewApp()
app.Flags = cliapp.ProtectFlags(server.Flags)
app.Flags = cliapp.ProtectFlags(flags.Flags)
app.Version = Version
app.Name = "eigenda-proxy"
app.Usage = "EigenDA Proxy Sidecar Service"
Expand Down
4 changes: 2 additions & 2 deletions e2e/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func TestKeccak256CommitmentRequestErrorsWhenS3NotSet(t *testing.T) {
testCfg.UseKeccak256ModeS3 = true

tsConfig := e2e.TestSuiteConfig(t, testCfg)
tsConfig.S3Config.Endpoint = ""
tsConfig.EigenDAConfig.S3Config.Endpoint = ""
ts, kill := e2e.CreateTestSuite(t, tsConfig)
defer kill()

Expand Down Expand Up @@ -393,7 +393,7 @@ func TestProxyServerCachingWithRedis(t *testing.T) {
require.Equal(t, testPreimage, preimage)

// ensure that read was from cache
redStats, err := ts.Server.GetStoreStats(store.Redis)
redStats, err := ts.Server.GetStoreStats(store.RedisBackendType)
require.NoError(t, err)

require.Equal(t, 1, redStats.Reads)
Expand Down
77 changes: 47 additions & 30 deletions e2e/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,19 @@ import (
"context"
"fmt"
"os"
"runtime"
"testing"
"time"

"github.com/Layr-Labs/eigenda-proxy/metrics"
"github.com/Layr-Labs/eigenda-proxy/server"
"github.com/Layr-Labs/eigenda-proxy/store"
"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
"github.com/Layr-Labs/eigenda-proxy/utils"
"github.com/Layr-Labs/eigenda-proxy/verify"
"github.com/Layr-Labs/eigenda/api/clients"
"github.com/Layr-Labs/eigenda/encoding/kzg"
"github.com/ethereum/go-ethereum/log"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
Expand Down Expand Up @@ -53,15 +59,15 @@ func TestConfig(useMemory bool) *Cfg {
}

func createRedisConfig(eigendaCfg server.Config) server.CLIConfig {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Knit - is eigendaCfg overly verbose?

eigendaCfg.RedisConfig = redis.Config{
Endpoint: "127.0.0.1:9001",
Password: "",
DB: 0,
Eviction: 10 * time.Minute,
Profile: true,
}
return server.CLIConfig{
EigenDAConfig: eigendaCfg,
RedisCfg: store.RedisConfig{
Endpoint: "127.0.0.1:9001",
Password: "",
DB: 0,
Eviction: 10 * time.Minute,
Profile: true,
},
}
}

Expand All @@ -70,18 +76,18 @@ func createS3Config(eigendaCfg server.Config) server.CLIConfig {
bucketName := "eigenda-proxy-test-" + RandString(10)
createS3Bucket(bucketName)

eigendaCfg.S3Config = s3.Config{
Profiling: true,
Bucket: bucketName,
Path: "",
Endpoint: "localhost:4566",
AccessKeySecret: "minioadmin",
AccessKeyID: "minioadmin",
CredentialType: s3.CredentialTypeStatic,
Backup: false,
}
return server.CLIConfig{
EigenDAConfig: eigendaCfg,
S3Config: store.S3Config{
Profiling: true,
Bucket: bucketName,
Path: "",
Endpoint: "localhost:4566",
AccessKeySecret: "minioadmin",
AccessKeyID: "minioadmin",
S3CredentialType: store.S3CredentialStatic,
Backup: false,
},
}
}

Expand All @@ -105,28 +111,39 @@ func TestSuiteConfig(t *testing.T, testCfg *Cfg) server.CLIConfig {
pollInterval = time.Minute * 1
}

maxBlobLengthBytes, err := utils.ParseBytesAmount("16mib")
require.NoError(t, err)
eigendaCfg := server.Config{
ClientConfig: clients.EigenDAClientConfig{
EdaClientConfig: clients.EigenDAClientConfig{
RPC: holeskyDA,
StatusQueryTimeout: time.Minute * 45,
StatusQueryRetryInterval: pollInterval,
DisableTLS: false,
SignerPrivateKeyHex: pk,
},
EthRPC: ethRPC,
SvcManagerAddr: "0xD4A7E1Bd8015057293f0D0A557088c286942e84b", // incompatible with non holeskly networks
CacheDir: "../resources/SRSTables",
G1Path: "../resources/g1.point",
MaxBlobLength: "16mib",
G2PowerOfTauPath: "../resources/g2.point.powerOf2",
PutBlobEncodingVersion: 0x00,
MemstoreEnabled: testCfg.UseMemory,
MemstoreBlobExpiration: testCfg.Expiration,
EthConfirmationDepth: 0,
VerifierConfig: verify.Config{
VerifyCerts: false,
RPCURL: ethRPC,
SvcManagerAddr: "0xD4A7E1Bd8015057293f0D0A557088c286942e84b", // incompatible with non holeskly networks
EthConfirmationDepth: 0,
KzgConfig: &kzg.KzgConfig{
G1Path: "../resources/g1.point",
G2PowerOf2Path: "../resources/g2.point.powerOf2",
CacheDir: "../resources/SRSTables",
SRSOrder: 268435456,
SRSNumberToLoad: maxBlobLengthBytes / 32,
NumWorker: uint64(runtime.GOMAXPROCS(0)),
},
},
MemstoreEnabled: testCfg.UseMemory,
MemstoreConfig: memstore.Config{
BlobExpiration: testCfg.Expiration,
MaxBlobSizeBytes: maxBlobLengthBytes,
},
}

if testCfg.UseMemory {
eigendaCfg.ClientConfig.SignerPrivateKeyHex = "0000000000000000000100000000000000000000000000000000000000000000"
eigendaCfg.EdaClientConfig.SignerPrivateKeyHex = "0000000000000000000100000000000000000000000000000000000000000000"
}

var cfg server.CLIConfig
Expand Down
121 changes: 121 additions & 0 deletions flags/eigendaflags/cli.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package eigendaflags

import (
"time"

"github.com/Layr-Labs/eigenda/api/clients"
"github.com/Layr-Labs/eigenda/api/clients/codecs"
"github.com/urfave/cli/v2"
)

// TODO: we should eventually move all of these flags into the eigenda repo

var (
DisperserRPCFlagName = withFlagPrefix("disperser-rpc")
StatusQueryRetryIntervalFlagName = withFlagPrefix("status-query-retry-interval")
StatusQueryTimeoutFlagName = withFlagPrefix("status-query-timeout")
DisableTLSFlagName = withFlagPrefix("disable-tls")
ResponseTimeoutFlagName = withFlagPrefix("response-timeout")
CustomQuorumIDsFlagName = withFlagPrefix("custom-quorum-ids")
SignerPrivateKeyHexFlagName = withFlagPrefix("signer-private-key-hex")
PutBlobEncodingVersionFlagName = withFlagPrefix("put-blob-encoding-version")
DisablePointVerificationModeFlagName = withFlagPrefix("disable-point-verification-mode")
WaitForFinalizationFlagName = withFlagPrefix("wait-for-finalization")
)

func withFlagPrefix(s string) string {
return "eigenda." + s
}

func withEnvPrefix(envPrefix, s string) []string {
return []string{envPrefix + "_EIGENDA_" + s}
}

// CLIFlags ... used for EigenDA client configuration
func CLIFlags(envPrefix, category string) []cli.Flag {
return []cli.Flag{
&cli.StringFlag{
Name: DisperserRPCFlagName,
Usage: "RPC endpoint of the EigenDA disperser.",
EnvVars: withEnvPrefix(envPrefix, "DISPERSER_RPC"),
Category: category,
},
&cli.DurationFlag{
Name: StatusQueryTimeoutFlagName,
Usage: "Duration to wait for a blob to finalize after being sent for dispersal. Default is 30 minutes.",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Knit - hasn't this value been changed given the recent decrease in bridging frequency?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a good question. Do right now bridging frequency is:

  • preprod: 1 min
  • testnet: 10 min (still lol... but guessing they'll change it to 3 min also eventually..?)
  • mainnet: 3 min

Finalization itself requires waiting for 24 minutes though. So not even sure how 30 minutes was enough.
Feel like the msg in this flag should also reflect whether the WaitForFinalization flag is true or false?

Value: 30 * time.Minute,
EnvVars: withEnvPrefix(envPrefix, "STATUS_QUERY_TIMEOUT"),
Category: category,
},
&cli.DurationFlag{
Name: StatusQueryRetryIntervalFlagName,
Usage: "Interval between retries when awaiting network blob finalization. Default is 5 seconds.",
Value: 5 * time.Second,
EnvVars: withEnvPrefix(envPrefix, "STATUS_QUERY_INTERVAL"),
Category: category,
},
&cli.BoolFlag{
Name: DisableTLSFlagName,
Usage: "Disable TLS for gRPC communication with the EigenDA disperser. Default is false.",
Value: false,
EnvVars: withEnvPrefix(envPrefix, "GRPC_DISABLE_TLS"),
Category: category,
},
&cli.DurationFlag{
Name: ResponseTimeoutFlagName,
Usage: "Total time to wait for a response from the EigenDA disperser. Default is 60 seconds.",
Value: 60 * time.Second,
EnvVars: withEnvPrefix(envPrefix, "RESPONSE_TIMEOUT"),
Category: category,
},
&cli.UintSliceFlag{
Name: CustomQuorumIDsFlagName,
Usage: "Custom quorum IDs for writing blobs. Should not include default quorums 0 or 1.",
Value: cli.NewUintSlice(),
EnvVars: withEnvPrefix(envPrefix, "CUSTOM_QUORUM_IDS"),
Category: category,
},
&cli.StringFlag{
Name: SignerPrivateKeyHexFlagName,
Usage: "Hex-encoded signer private key. This key should not be associated with an Ethereum address holding any funds.",
EnvVars: withEnvPrefix(envPrefix, "SIGNER_PRIVATE_KEY_HEX"),
Category: category,
},
&cli.UintFlag{
Name: PutBlobEncodingVersionFlagName,
Usage: "Blob encoding version to use when writing blobs from the high-level interface.",
EnvVars: withEnvPrefix(envPrefix, "PUT_BLOB_ENCODING_VERSION"),
Value: 0,
Category: category,
},
&cli.BoolFlag{
Name: DisablePointVerificationModeFlagName,
Usage: "Disable point verification mode. This mode performs IFFT on data before writing and FFT on data after reading. Disabling requires supplying the entire blob for verification against the KZG commitment.",
EnvVars: withEnvPrefix(envPrefix, "DISABLE_POINT_VERIFICATION_MODE"),
Value: false,
Category: category,
},
&cli.BoolFlag{
Name: WaitForFinalizationFlagName,
Usage: "Wait for blob finalization before returning from PutBlob.",
EnvVars: withEnvPrefix(envPrefix, "WAIT_FOR_FINALIZATION"),
Value: false,
Category: category,
},
}
}

func ReadConfig(ctx *cli.Context) clients.EigenDAClientConfig {
return clients.EigenDAClientConfig{
RPC: ctx.String(DisperserRPCFlagName),
StatusQueryRetryInterval: ctx.Duration(StatusQueryRetryIntervalFlagName),
StatusQueryTimeout: ctx.Duration(StatusQueryTimeoutFlagName),
DisableTLS: ctx.Bool(DisableTLSFlagName),
ResponseTimeout: ctx.Duration(ResponseTimeoutFlagName),
CustomQuorumIDs: ctx.UintSlice(CustomQuorumIDsFlagName),
SignerPrivateKeyHex: ctx.String(SignerPrivateKeyHexFlagName),
PutBlobEncodingVersion: codecs.BlobEncodingVersion(ctx.Uint(PutBlobEncodingVersionFlagName)),
DisablePointVerificationMode: ctx.Bool(DisablePointVerificationModeFlagName),
WaitForFinalization: ctx.Bool(WaitForFinalizationFlagName),
}
}
Loading
Loading