diff --git a/prover/prover.go b/prover/prover.go
index 22c1cecf6..452dd6b5f 100644
--- a/prover/prover.go
+++ b/prover/prover.go
@@ -127,16 +127,6 @@ func InitFromConfig(ctx context.Context, p *Prover, cfg *Config) (err error) {
 		return err
 	}
 
-	// Prover server
-	if p.srv, err = server.New(&server.NewProverServerOpts{
-		ProverPrivateKey: p.cfg.L1ProverPrivKey,
-		MinProofFee:      p.cfg.MinProofFee,
-		MaxExpiry:        p.cfg.MaxExpiry,
-		CapacityManager:  p.capacityManager,
-	}); err != nil {
-		return err
-	}
-
 	// Configs
 	protocolConfigs, err := p.rpc.TaikoL1.GetConfig(&bind.CallOpts{Context: ctx})
 	if err != nil {
@@ -214,6 +204,20 @@ func InitFromConfig(ctx context.Context, p *Prover, cfg *Config) (err error) {
 		return err
 	}
 
+	// Prover server
+	proverServerOpts := &server.NewProverServerOpts{
+		ProverPrivateKey: p.cfg.L1ProverPrivKey,
+		MinProofFee:      p.cfg.MinProofFee,
+		MaxExpiry:        p.cfg.MaxExpiry,
+		CapacityManager:  p.capacityManager,
+	}
+	if p.cfg.OracleProver {
+		proverServerOpts.ProverPrivateKey = p.cfg.OracleProverPrivateKey
+	}
+	if p.srv, err = server.New(proverServerOpts); err != nil {
+		return err
+	}
+
 	return nil
 }