Support Tile Clusters (tree-like hierarchies)

src/main/scala/devices/debug/Debug.scala

@@ -690,7 +690,8 @@ class TLDebugModuleOuterAsync(device: Device)(implicit p: Parameters) extends La
   })
 
   val dmOuter = LazyModule( new TLDebugModuleOuter(device))
-  val intnode = IntSyncCrossingSource(alreadyRegistered = true) :*= dmOuter.intnode
+  val intnode = IntSyncIdentityNode()
+  intnode :*= IntSyncCrossingSource(alreadyRegistered = true) :*= dmOuter.intnode
 
   val dmiBypass = LazyModule(new TLBusBypass(beatBytes=4, bufferError=false, maxAtomic=0, maxTransfer=4))
   val dmiInnerNode = TLAsyncCrossingSource() := dmiBypass.node := dmiXbar.node

src/main/scala/devices/debug/Periphery.scala

@@ -97,9 +97,10 @@ trait HasPeripheryDebug { this: BaseSubsystem =>
     }
     tlDM
   }
+  val debugNode = debugOpt.map(_.intnode)
 }
 
-trait HasPeripheryDebugModuleImp extends LazyModuleImp {
+trait HasPeripheryDebugModuleImp extends LazyRawModuleImp {
   val outer: HasPeripheryDebug
 
   val psd = IO(new PSDIO)

src/main/scala/devices/tilelink/BootROM.scala

@@ -5,7 +5,7 @@ package freechips.rocketchip.devices.tilelink
 import chisel3._
 import chisel3.util.log2Ceil
 import freechips.rocketchip.config.{Field, Parameters}
-import freechips.rocketchip.subsystem.{BaseSubsystem, HierarchicalLocation, HasTiles, TLBusWrapperLocation}
+import freechips.rocketchip.subsystem._
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.prci.{ClockSinkDomain}
@@ -66,7 +66,7 @@ object BootROM {
     *    at a configurable location, but also drives the tiles' reset vectors to point
     *    at its 'hang' address parameter value.
     */
-  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasTiles, where: TLBusWrapperLocation)
+  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasElements with HasTileInputConstants, where: TLBusWrapperLocation)
             (implicit p: Parameters): TLROM = {
     val tlbus = subsystem.locateTLBusWrapper(where)
     val bootROMDomainWrapper = LazyModule(new ClockSinkDomain(take = None))

src/main/scala/devices/tilelink/CLINT.scala

@@ -116,4 +116,5 @@ trait CanHavePeripheryCLINT { this: BaseSubsystem =>
     clint
 
   }
+  val clintNode = clintOpt.map(_.intnode)
 }

src/main/scala/devices/tilelink/Plic.scala

@@ -365,4 +365,5 @@ trait CanHavePeripheryPLIC { this: BaseSubsystem =>
 
     plic
   }
+  val plicNode = plicOpt.map(_.intnode)
 }

src/main/scala/groundtest/Configs.scala

@@ -30,25 +30,24 @@ class GroundTestBaseConfig extends Config(
     case DebugModuleKey => None
     case CLINTKey => None
     case PLICKey => None
-    case SubsystemExternalResetVectorKey => true
+    case HasTilesExternalResetVectorKey => true
   })
 )
 
 class WithTraceGen(
   n: Int = 2,
-  overrideIdOffset: Option[Int] = None,
   overrideMemOffset: Option[BigInt] = None)(
   params: Seq[DCacheParams] = List.fill(n){ DCacheParams(nSets = 16, nWays = 1) },
   nReqs: Int = 8192
 ) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     val memOffset: BigInt = overrideMemOffset.orElse(site(ExtMem).map(_.master.base)).getOrElse(0x0L)
     params.zipWithIndex.map { case (dcp, i) =>
       TraceGenTileAttachParams(
         tileParams = TraceGenParams(
-          hartId = i + idOffset,
+          tileId = i + idOffset,
           dcache = Some(dcp),
           wordBits = site(XLen),
           addrBits = 32,
@@ -68,4 +67,5 @@ class WithTraceGen(
       )
     } ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })

src/main/scala/groundtest/GroundTestSubsystem.scala

@@ -4,14 +4,22 @@ package freechips.rocketchip.groundtest
 
 import chisel3._
 import freechips.rocketchip.config.{Parameters}
-import freechips.rocketchip.diplomacy.{AddressSet, LazyModule}
-import freechips.rocketchip.interrupts.{IntSinkNode, IntSinkPortSimple}
-import freechips.rocketchip.subsystem.{BaseSubsystem, BaseSubsystemModuleImp, HasTiles, CanHaveMasterAXI4MemPort}
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.interrupts._
+import freechips.rocketchip.tile.{NMI}
+import freechips.rocketchip.devices.tilelink.{CLINTConsts}
+import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tilelink.{TLRAM, TLFragmenter}
+import freechips.rocketchip.rocket.{TracedInstruction}
+import freechips.rocketchip.util.{TraceCoreInterface}
 
 class GroundTestSubsystem(implicit p: Parameters)
   extends BaseSubsystem
-  with HasTiles
+  with InstantiatesElements
+  with HasElementsRootContext
+  with HasElements
+  with HasTileNotificationSinks
+  with HasTileInputConstants
   with CanHaveMasterAXI4MemPort
 {
   val testram = LazyModule(new TLRAM(AddressSet(0x52000000, 0xfff), beatBytes=pbus.beatBytes))
@@ -23,13 +31,16 @@ class GroundTestSubsystem(implicit p: Parameters)
   // No PLIC in ground test; so just sink the interrupts to nowhere
   IntSinkNode(IntSinkPortSimple()) :=* ibus.toPLIC
 
-  val tileStatusNodes = tiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
+  val tileStatusNodes = totalTiles.values.collect { case t: GroundTestTile => t.statusNode.makeSink() }
+  val clintOpt = None
+  val debugOpt = None
+  val plicOpt = None
 
   override lazy val module = new GroundTestSubsystemModuleImp(this)
 }
 
 class GroundTestSubsystemModuleImp[+L <: GroundTestSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer) {
   val success = IO(Output(Bool()))
-  val status = dontTouch(DebugCombiner(outer.tileStatusNodes.map(_.bundle)))
+  val status = dontTouch(DebugCombiner(outer.tileStatusNodes.map(_.bundle).toSeq))
   success := outer.tileCeaseSinkNode.in.head._1.asUInt.andR
 }

src/main/scala/groundtest/Tile.scala

@@ -34,7 +34,7 @@ abstract class GroundTestTile(
   with SourcesExternalNotifications
 {
   val cpuDevice: SimpleDevice = new SimpleDevice("groundtest", Nil)
-  val intOutwardNode: IntOutwardNode = IntIdentityNode()
+  val intOutwardNode = None
   val slaveNode: TLInwardNode = TLIdentityNode()
   val statusNode = BundleBridgeSource(() => new GroundTestStatus)
 

src/main/scala/groundtest/TraceGen.scala

@@ -26,7 +26,7 @@ import freechips.rocketchip.diplomacy.{ClockCrossingType}
 import freechips.rocketchip.rocket._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.subsystem.{TileCrossingParamsLike, CanAttachTile}
+import freechips.rocketchip.subsystem.{ElementCrossingParamsLike, CanAttachTile}
 import freechips.rocketchip.util._
 import freechips.rocketchip.prci.{ClockSinkParameters}
 
@@ -68,15 +68,15 @@ case class TraceGenParams(
     memStart: BigInt, //p(ExtMem).base
     numGens: Int,
     dcache: Option[DCacheParams] = Some(DCacheParams()),
-    hartId: Int = 0
+    tileId: Int = 0
 ) extends InstantiableTileParams[TraceGenTile] with GroundTestTileParams
 {
-  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
+  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
     new TraceGenTile(this, crossing, lookup)
   }
-  val beuAddr = None
   val blockerCtrlAddr = None
-  val name = None
+  val baseName = "tracegentile"
+  val uniqueName = s"${baseName}_$tileId"
   val clockSinkParams = ClockSinkParameters()
 }
 
@@ -105,7 +105,7 @@ trait HasTraceGenParams {
 
 case class TraceGenTileAttachParams(
   tileParams: TraceGenParams,
-  crossingParams: TileCrossingParamsLike
+  crossingParams: ElementCrossingParamsLike
 ) extends CanAttachTile {
   type TileType = TraceGenTile
   val lookup: LookupByHartIdImpl = HartsWontDeduplicate(tileParams)
@@ -617,7 +617,7 @@ class TraceGenTile private(
   q: Parameters
 ) extends GroundTestTile(params, crossing, lookup, q)
 {
-  def this(params: TraceGenParams, crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: TraceGenParams, crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   val masterNode: TLOutwardNode = TLIdentityNode() := visibilityNode := dcacheOpt.map(_.node).getOrElse(TLTempNode())
@@ -644,5 +644,5 @@ class TraceGenTileModuleImp(outer: TraceGenTile) extends GroundTestTileModuleImp
   status.timeout.bits := 0.U
   status.error.valid := false.B
 
-  assert(!tracegen.io.timeout, s"TraceGen tile ${outer.tileParams.hartId}: request timed out")
+  assert(!tracegen.io.timeout, s"TraceGen tile ${outer.tileParams.tileId}: request timed out")
 }

src/main/scala/interrupts/package.scala

@@ -12,6 +12,10 @@ package object interrupts
   type IntOutwardNode = OutwardNodeHandle[IntSourcePortParameters, IntSinkPortParameters, IntEdge, Vec[Bool]]
   type IntNode = SimpleNodeHandle[IntSourcePortParameters, IntSinkPortParameters, IntEdge, Vec[Bool]]
 
+  type IntSyncInwardNode = InwardNodeHandle[IntSourcePortParameters, IntSinkPortParameters, IntEdge, SyncInterrupts]
+  type IntSyncOutwardNode = OutwardNodeHandle[IntSourcePortParameters, IntSinkPortParameters, IntEdge, SyncInterrupts]
+  type IntSyncNode = SimpleNodeHandle[IntSourcePortParameters, IntSinkPortParameters, IntEdge, SyncInterrupts]
+
   implicit class IntClockDomainCrossing(private val x: HasClockDomainCrossing) extends AnyVal {
     def crossIn (n: IntInwardNode) (implicit valName: ValName) = IntInwardClockCrossingHelper(valName.name, x, n)
     def crossOut(n: IntOutwardNode)(implicit valName: ValName) = IntOutwardClockCrossingHelper(valName.name, x, n)

src/main/scala/prci/ClockBundles.scala

@@ -5,7 +5,7 @@ import chisel3._
 import freechips.rocketchip.util.RecordMap
 
 
-class ClockBundle(val params: ClockBundleParameters) extends Bundle
+class ClockBundle(val params: ClockBundleParameters = ClockBundleParameters()) extends Bundle
 {
   val clock = Output(Clock())
   val reset = Output(Reset())

src/main/scala/prci/ClockDomain.scala

@@ -4,7 +4,8 @@ import chisel3._
 import freechips.rocketchip.config.Parameters
 import freechips.rocketchip.diplomacy._
 
-abstract class Domain(implicit p: Parameters) extends LazyModule with HasDomainCrossing {
+abstract class Domain(implicit p: Parameters) extends LazyModule with HasDomainCrossing
+{
   def clockBundle: ClockBundle
 
   lazy val module = new Impl

src/main/scala/prci/ClockGroup.scala

@@ -4,7 +4,7 @@ package freechips.rocketchip.prci
 import freechips.rocketchip.config.Parameters
 import freechips.rocketchip.diplomacy._
 
-case class ClockGroupNode(groupName: String)(implicit valName: ValName)
+case class ClockGroupingNode(groupName: String)(implicit valName: ValName)
   extends MixedNexusNode(ClockGroupImp, ClockImp)(
     dFn = { _ => ClockSourceParameters() },
     uFn = { seq => ClockGroupSinkParameters(name = groupName, members = seq) })
@@ -14,7 +14,7 @@ case class ClockGroupNode(groupName: String)(implicit valName: ValName)
 
 class ClockGroup(groupName: String)(implicit p: Parameters) extends LazyModule
 {
-  val node = ClockGroupNode(groupName)
+  val node = ClockGroupingNode(groupName)
 
   lazy val module = new Impl
   class Impl extends LazyRawModuleImp(this) {
@@ -107,7 +107,7 @@ class FixedClockBroadcast(fixedClockOpt: Option[ClockParameters])(implicit p: Pa
 
 object FixedClockBroadcast
 {
-  def apply(fixedClockOpt: Option[ClockParameters])(implicit p: Parameters, valName: ValName) = LazyModule(new FixedClockBroadcast(fixedClockOpt)).node
+  def apply(fixedClockOpt: Option[ClockParameters] = None)(implicit p: Parameters, valName: ValName) = LazyModule(new FixedClockBroadcast(fixedClockOpt)).node
 }
 
 case class PRCIClockGroupNode()(implicit valName: ValName)

src/main/scala/prci/package.scala

@@ -9,6 +9,9 @@ package object prci
   type ClockInwardNode = InwardNodeHandle[ClockSourceParameters, ClockSinkParameters, ClockEdgeParameters, ClockBundle]
   type ClockOutwardNode = OutwardNodeHandle[ClockSourceParameters, ClockSinkParameters, ClockEdgeParameters, ClockBundle]
   type ClockNode = NodeHandle[ClockSourceParameters, ClockSinkParameters, ClockEdgeParameters, ClockBundle, ClockSourceParameters, ClockSinkParameters, ClockEdgeParameters, ClockBundle]
+
+  type ClockGroupNode = NodeHandle[ClockGroupSourceParameters, ClockGroupSinkParameters, ClockGroupEdgeParameters, ClockGroupBundle, ClockGroupSourceParameters, ClockGroupSinkParameters, ClockGroupEdgeParameters, ClockGroupBundle]
+
   def asyncMux[T](xType: ClockCrossingType, async: T, notasync: T): T = xType match {
     case _: AsynchronousCrossing => async
     case _ => notasync

src/main/scala/rocket/CSR.scala

@@ -239,10 +239,10 @@ class CSRDecodeIO(implicit p: Parameters) extends CoreBundle {
   val virtual_system_illegal = Output(Bool())
 }
 
-class CSRFileIO(implicit p: Parameters) extends CoreBundle
+class CSRFileIO(hasBeu: Boolean)(implicit p: Parameters) extends CoreBundle
     with HasCoreParameters {
   val ungated_clock = Input(Clock())
-  val interrupts = Input(new CoreInterrupts())
+  val interrupts = Input(new CoreInterrupts(hasBeu))
   val hartid = Input(UInt(hartIdLen.W))
   val rw = new Bundle {
     val addr = Input(UInt(CSR.ADDRSZ.W))
@@ -357,10 +357,11 @@ class VType(implicit p: Parameters) extends CoreBundle {
 
 class CSRFile(
   perfEventSets: EventSets = new EventSets(Seq()),
-  customCSRs: Seq[CustomCSR] = Nil)(implicit p: Parameters)
+  customCSRs: Seq[CustomCSR] = Nil,
+  hasBeu: Boolean = false)(implicit p: Parameters)
     extends CoreModule()(p)
     with HasCoreParameters {
-  val io = IO(new CSRFileIO {
+  val io = IO(new CSRFileIO(hasBeu) {
     val customCSRs = Output(Vec(CSRFile.this.customCSRs.size, new CustomCSRIO))
   })
 

src/main/scala/rocket/Frontend.scala

@@ -60,9 +60,9 @@ class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) {
   val progress = Output(Bool())
 }
 
-class Frontend(val icacheParams: ICacheParams, staticIdForMetadataUseOnly: Int)(implicit p: Parameters) extends LazyModule {
+class Frontend(val icacheParams: ICacheParams, tileId: Int)(implicit p: Parameters) extends LazyModule {
   lazy val module = new FrontendModule(this)
-  val icache = LazyModule(new ICache(icacheParams, staticIdForMetadataUseOnly))
+  val icache = LazyModule(new ICache(icacheParams, tileId))
   val masterNode = icache.masterNode
   val slaveNode = icache.slaveNode
   val resetVectorSinkNode = BundleBridgeSink[UInt](Some(() => UInt(masterNode.edges.out.head.bundle.addressBits.W)))
@@ -382,7 +382,7 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
 /** Mix-ins for constructing tiles that have an ICache-based pipeline frontend */
 trait HasICacheFrontend extends CanHavePTW { this: BaseTile =>
   val module: HasICacheFrontendModule
-  val frontend = LazyModule(new Frontend(tileParams.icache.get, staticIdForMetadataUseOnly))
+  val frontend = LazyModule(new Frontend(tileParams.icache.get, tileId))
   tlMasterXbar.node := frontend.masterNode
   connectTLSlave(frontend.slaveNode, tileParams.core.fetchBytes)
   frontend.icache.hartIdSinkNodeOpt.foreach { _ := hartIdNexusNode }

src/main/scala/rocket/HellaCache.scala

@@ -188,17 +188,17 @@ class HellaCacheIO(implicit p: Parameters) extends CoreBundle()(p) {
 
 /** Base classes for Diplomatic TL2 HellaCaches */
 
-abstract class HellaCache(staticIdForMetadataUseOnly: Int)(implicit p: Parameters) extends LazyModule
+abstract class HellaCache(tileId: Int)(implicit p: Parameters) extends LazyModule
     with HasNonDiplomaticTileParameters {
   protected val cfg = tileParams.dcache.get
 
   protected def cacheClientParameters = cfg.scratch.map(x => Seq()).getOrElse(Seq(TLMasterParameters.v1(
-    name          = s"Core ${staticIdForMetadataUseOnly} DCache",
+    name          = s"Core ${tileId} DCache",
     sourceId      = IdRange(0, 1 max cfg.nMSHRs),
     supportsProbe = TransferSizes(cfg.blockBytes, cfg.blockBytes))))
 
   protected def mmioClientParameters = Seq(TLMasterParameters.v1(
-    name          = s"Core ${staticIdForMetadataUseOnly} DCache MMIO",
+    name          = s"Core ${tileId} DCache MMIO",
     sourceId      = IdRange(firstMMIO, firstMMIO + cfg.nMMIOs),
     requestFifo   = true))
 
@@ -252,9 +252,9 @@ case object BuildHellaCache extends Field[BaseTile => Parameters => HellaCache](
 object HellaCacheFactory {
   def apply(tile: BaseTile)(p: Parameters): HellaCache = {
     if (tile.tileParams.dcache.get.nMSHRs == 0)
-      new DCache(tile.staticIdForMetadataUseOnly, tile.crossing)(p)
+      new DCache(tile.tileId, tile.crossing)(p)
     else
-      new NonBlockingDCache(tile.staticIdForMetadataUseOnly)(p)
+      new NonBlockingDCache(tile.tileId)(p)
   }
 }