feat(SoC): Replace DummyLLC with OpenLLC+OpenNCB in KunminghuV2Config

Makefile

@@ -198,6 +198,7 @@ clean:
 init:
 	git submodule update --init
 	cd rocket-chip && git submodule update --init cde hardfloat
+	cd openLLC && git submodule update --init openNCB
 
 bump:
 	git submodule foreach "git fetch origin&&git checkout master&&git reset --hard origin/master"

build.sc

@@ -143,6 +143,16 @@ object coupledL2 extends millbuild.coupledL2.common.CoupledL2Module with HasChis
 
 }
 
+object openNCB extends SbtModule with HasChisel {
+
+  override def millSourcePath = os.pwd / "openLLC" / "openNCB"
+
+  override def moduleDeps = super.moduleDeps ++ Seq(
+    rocketchip
+  )
+
+}
+
 object openLLC extends millbuild.openLLC.common.OpenLLCModule with HasChisel {
 
   override def millSourcePath = os.pwd / "openLLC"
@@ -152,6 +162,9 @@ object openLLC extends millbuild.openLLC.common.OpenLLCModule with HasChisel {
   def rocketModule: ScalaModule = rocketchip
 
   def utilityModule: ScalaModule = utility
+
+  def openNCBModule: ScalaModule = openNCB
+
 }
 
 object difftest extends HasChisel {

src/main/scala/system/SoC.scala

@@ -32,8 +32,9 @@ import top.BusPerfMonitor
 import utility.{ReqSourceKey, TLClientsMerger, TLEdgeBuffer, TLLogger}
 import xiangshan.backend.fu.PMAConst
 import xiangshan.{DebugOptionsKey, XSTileKey}
-import coupledL2.EnableCHI
+import coupledL2.{EnableCHI, L2Param}
 import coupledL2.tl2chi.CHIIssue
+import openLLC.OpenLLCParam
 
 case object SoCParamsKey extends Field[SoCParameters]
 
@@ -50,6 +51,13 @@ case class SoCParameters
     ways = 8,
     sets = 2048 // 1MB per bank
   )),
+  OpenLLCParamsOpt: Option[OpenLLCParam] = Some(OpenLLCParam(
+    name = "LLC",
+    ways = 8,
+    sets = 2048,
+    banks = 4,
+    clientCaches = Seq(L2Param())
+  )),
   XSTopPrefix: Option[String] = None,
   NodeIDWidthList: Map[String, Int] = Map(
     "B" -> 7,

src/main/scala/top/Configs.scala

@@ -31,6 +31,7 @@ import system._
 import utility._
 import utils._
 import huancun._
+import openLLC.{OpenLLCParam}
 import xiangshan._
 import xiangshan.backend.dispatch.DispatchParameters
 import xiangshan.backend.regfile.{IntPregParams, VfPregParams}
@@ -340,6 +341,17 @@ class WithNKBL3(n: Int, ways: Int = 8, inclusive: Boolean = true, banks: Int = 1
         simulation = !site(DebugOptionsKey).FPGAPlatform,
         prefetch = Some(huancun.prefetch.L3PrefetchReceiverParams()),
         tpmeta = Some(huancun.prefetch.DefaultTPmetaParameters())
+      )),
+      OpenLLCParamsOpt = Some(OpenLLCParam(
+        name = "LLC",
+        ways = ways,
+        sets = sets,
+        banks = banks,
+        fullAddressBits = 48,
+        clientCaches = tiles.map { core =>
+          val l2params = core.L2CacheParamsOpt.get
+          l2params.copy(sets = 2 * clientDirBytes / core.L2NBanks / l2params.ways / 64, ways = l2params.ways + 2)
+        }
       ))
     )
 })

src/main/scala/top/Top.scala

@@ -25,7 +25,10 @@ import xiangshan._
 import utils._
 import huancun.{HCCacheParameters, HCCacheParamsKey, HuanCun, PrefetchRecv, TPmetaResp}
 import coupledL2.EnableCHI
-import openLLC.DummyLLC
+import coupledL2.tl2chi.CHILogger
+import openLLC.{OpenLLC, OpenLLCParamKey, OpenNCB}
+import openLLC.TargetBinder._
+import cc.xiangshan.openncb._
 import utility._
 import system._
 import device._
@@ -38,6 +41,7 @@ import freechips.rocketchip.amba.axi4._
 import freechips.rocketchip.jtag.JTAGIO
 import chisel3.experimental.{annotate, ChiselAnnotation}
 import sifive.enterprise.firrtl.NestedPrefixModulesAnnotation
+import scala.collection.mutable.{Map}
 
 abstract class BaseXSSoc()(implicit p: Parameters) extends LazyModule
   with BindingScope
@@ -95,7 +99,33 @@ class XSTop()(implicit p: Parameters) extends BaseXSSoc() with HasSoCParameter
     })))
   )
 
-  val chi_dummyllc_opt = Option.when(enableCHI)(LazyModule(new DummyLLC(numRNs = NumCores)(p)))
+  val chi_llcBridge_opt = Option.when(enableCHI)(
+    LazyModule(new OpenNCB()(p.alter((site, here, up) => {
+      case NCBParametersKey => new NCBParameters(
+        axiMasterOrder      = EnumAXIMasterOrder.WriteAddress,
+        readCompDMT         = false,
+        writeCancelable     = false,
+        writeNoError        = true,
+        axiBurstAlwaysIncr  = true
+      )
+    })))
+  )
+
+  val chi_mmioBridge_opt = Seq.fill(NumCores)(Option.when(enableCHI)(
+    LazyModule(new OpenNCB()(p.alter((site, here, up) => {
+      case NCBParametersKey => new NCBParameters(
+        axiMasterOrder              = EnumAXIMasterOrder.None,
+        readCompDMT                 = false,
+        writeCancelable             = false,
+        writeNoError                = true,
+        asEndpoint                  = false,
+        acceptOrderEndpoint         = true,
+        acceptMemAttrDevice         = true,
+        readReceiptAfterAcception   = true,
+        axiBurstAlwaysIncr          = true
+      )
+    })))
+  ))
 
   // receive all prefetch req from cores
   val memblock_pf_recv_nodes: Seq[Option[BundleBridgeSink[PrefetchRecv]]] = core_with_l2.map(_.core_l3_pf_port).map{
@@ -161,12 +191,20 @@ class XSTop()(implicit p: Parameters) extends BaseXSSoc() with HasSoCParameter
     case None =>
   }
 
-  chi_dummyllc_opt match {
-    case Some(llc) =>
-      misc.soc_xbar.get := llc.axi4node
+  chi_llcBridge_opt match {
+    case Some(ncb) =>
+      misc.soc_xbar.get := ncb.axi4node
     case None =>
   }
 
+  chi_mmioBridge_opt.foreach { e =>
+    e match {
+      case Some(ncb) =>
+        misc.soc_xbar.get := ncb.axi4node
+      case None =>
+    }
+  }
+
   class XSTopImp(wrapper: LazyModule) extends LazyRawModuleImp(wrapper) {
     soc.XSTopPrefix.foreach { prefix =>
       val mod = this.toNamed
@@ -217,6 +255,13 @@ class XSTop()(implicit p: Parameters) extends BaseXSSoc() with HasSoCParameter
 
     val reset_sync = withClockAndReset(io.clock.asClock, io.reset) { ResetGen() }
     val jtag_reset_sync = withClockAndReset(io.systemjtag.jtag.TCK, io.systemjtag.reset) { ResetGen() }
+    val chi_openllc_opt = Option.when(enableCHI)(
+      withClockAndReset(io.clock.asClock, io.reset) {
+        Module(new OpenLLC()(p.alter((site, here, up) => {
+          case OpenLLCParamKey => soc.OpenLLCParamsOpt.get
+        })))
+      }
+    )
 
     // override LazyRawModuleImp's clock and reset
     childClock := io.clock.asClock
@@ -244,9 +289,28 @@ class XSTop()(implicit p: Parameters) extends BaseXSSoc() with HasSoCParameter
       core.module.io.clintTime := misc.module.clintTime
       io.riscv_halt(i) := core.module.io.cpu_halt
       core.module.io.reset_vector := io.riscv_rst_vec(i)
-      chi_dummyllc_opt.foreach { case llc =>
-        llc.module.io.rn(i) <> core.module.io.chi.get
-        core.module.io.nodeID.get := i.U // TODO
+    }
+
+    withClockAndReset(io.clock.asClock, io.reset) {
+      Option.when(enableCHI)(true.B).foreach { _ =>
+        for ((core, i) <- core_with_l2.zipWithIndex) {
+          val mmioLogger = CHILogger(s"L2[${i}]_MMIO", true)
+          val llcLogger = CHILogger(s"L2[${i}]_LLC", true)
+          dontTouch(core.module.io.chi.get)
+          bind(
+            route(
+              core.module.io.chi.get, Map((AddressSet(0x0L, 0x00007fffffffL), NumCores + i)) ++ AddressSet(0x0L,
+              0xffffffffffffL).subtract(AddressSet(0x0L, 0x00007fffffffL)).map(addr => (addr, NumCores * 2)).toMap
+            ),
+            Map((NumCores + i) -> mmioLogger.io.up, (NumCores * 2) -> llcLogger.io.up)
+          )
+          chi_mmioBridge_opt(i).get.module.io.chi.connect(mmioLogger.io.down)
+          chi_openllc_opt.get.io.rn(i) <> llcLogger.io.down
+        }
+        val memLogger = CHILogger(s"LLC_MEM", true)
+        chi_openllc_opt.get.io.sn.connect(memLogger.io.up)
+        chi_llcBridge_opt.get.module.io.chi.connect(memLogger.io.down)
+        chi_openllc_opt.get.io.nodeID := (NumCores * 2).U
       }
     }