diff --git a/core/cache_subsystem/axi_adapter.sv b/core/cache_subsystem/axi_adapter.sv index e5443735ba..c206a38176 100644 --- a/core/cache_subsystem/axi_adapter.sv +++ b/core/cache_subsystem/axi_adapter.sv @@ -23,6 +23,7 @@ module axi_adapter #( parameter int unsigned AXI_ADDR_WIDTH = 0, parameter int unsigned AXI_DATA_WIDTH = 0, parameter int unsigned AXI_ID_WIDTH = 0, + parameter int unsigned MAX_OUTSTANDING_AW = 0, parameter type axi_req_t = ariane_axi::req_t, parameter type axi_rsp_t = ariane_axi::resp_t )( @@ -52,6 +53,9 @@ module axi_adapter #( ); localparam BURST_SIZE = (DATA_WIDTH/AXI_DATA_WIDTH)-1; localparam ADDR_INDEX = ($clog2(DATA_WIDTH/AXI_DATA_WIDTH) > 0) ? $clog2(DATA_WIDTH/AXI_DATA_WIDTH) : 1; + localparam MAX_OUTSTANDING_AW_CNT_WIDTH = $clog2(MAX_OUTSTANDING_AW + 1) > 0 ? $clog2(MAX_OUTSTANDING_AW + 1) : 1; + + typedef logic [MAX_OUTSTANDING_AW_CNT_WIDTH-1:0] outstanding_aw_cnt_t; enum logic [3:0] { IDLE, WAIT_B_VALID, WAIT_AW_READY, WAIT_LAST_W_READY, WAIT_LAST_W_READY_AW_READY, WAIT_AW_READY_BURST, @@ -68,6 +72,11 @@ module axi_adapter #( // save the atomic operation and size ariane_pkg::amo_t amo_d, amo_q; logic [1:0] size_d, size_q; + // outstanding write transactions counter + outstanding_aw_cnt_t outstanding_aw_cnt_q, outstanding_aw_cnt_d; + logic any_outstanding_aw; + + assign any_outstanding_aw = outstanding_aw_cnt_q != '0; always_comb begin : axi_fsm // Default assignments @@ -131,6 +140,8 @@ module axi_adapter #( size_d = size_q; index = '0; + outstanding_aw_cnt_d = outstanding_aw_cnt_q; + case (state_q) IDLE: begin @@ -140,70 +151,80 @@ module axi_adapter #( // is this a read or write? // write if (we_i) begin - // the data is valid - axi_req_o.aw_valid = 1'b1; - axi_req_o.w_valid = 1'b1; - // store-conditional requires exclusive access - axi_req_o.aw.lock = amo_i == ariane_pkg::AMO_SC; - // its a single write - if (type_i == ariane_axi::SINGLE_REQ) begin - // only a single write so the data is already the last one - axi_req_o.w.last = 1'b1; - // single req can be granted here - gnt_o = axi_resp_i.aw_ready & axi_resp_i.w_ready; - case ({axi_resp_i.aw_ready, axi_resp_i.w_ready}) - 2'b11: state_d = WAIT_B_VALID; - 2'b01: state_d = WAIT_AW_READY; - 2'b10: state_d = WAIT_LAST_W_READY; - default: state_d = IDLE; - endcase - - if (axi_resp_i.aw_ready) begin - amo_d = amo_i; - size_d = size_i; + // multiple outstanding write transactions are only + // allowed if they are guaranteed not to be reordered + // i.e. same ID + if (!any_outstanding_aw || ((id_i == id_q) && (amo_i == ariane_pkg::AMO_NONE))) begin + // the data is valid + axi_req_o.aw_valid = 1'b1; + axi_req_o.w_valid = 1'b1; + // store-conditional requires exclusive access + axi_req_o.aw.lock = amo_i == ariane_pkg::AMO_SC; + // its a single write + if (type_i == ariane_axi::SINGLE_REQ) begin + // only a single write so the data is already the last one + axi_req_o.w.last = 1'b1; + // single req can be granted here + gnt_o = axi_resp_i.aw_ready & axi_resp_i.w_ready; + case ({axi_resp_i.aw_ready, axi_resp_i.w_ready}) + 2'b11: state_d = WAIT_B_VALID; + 2'b01: state_d = WAIT_AW_READY; + 2'b10: state_d = WAIT_LAST_W_READY; + default: state_d = IDLE; + endcase + + if (axi_resp_i.aw_ready) begin + id_d = id_i; + amo_d = amo_i; + size_d = size_i; + end + + // its a request for the whole cache line + end else begin + // bursts of AMOs unsupported + assert (amo_i == ariane_pkg::AMO_NONE) + else $fatal("Bursts of atomic operations are not supported"); + + axi_req_o.aw.len = BURST_SIZE; // number of bursts to do + axi_req_o.w.data = wdata_i[0]; + axi_req_o.w.strb = be_i[0]; + + if (axi_resp_i.w_ready) + cnt_d = BURST_SIZE - 1; + else + cnt_d = BURST_SIZE; + + case ({axi_resp_i.aw_ready, axi_resp_i.w_ready}) + 2'b11: state_d = WAIT_LAST_W_READY; + 2'b01: state_d = WAIT_LAST_W_READY_AW_READY; + 2'b10: state_d = WAIT_LAST_W_READY; + default:; + endcase end - - // its a request for the whole cache line - end else begin - // bursts of AMOs unsupported - assert (amo_i == ariane_pkg::AMO_NONE) - else $fatal("Bursts of atomic operations are not supported"); - - axi_req_o.aw.len = BURST_SIZE; // number of bursts to do - axi_req_o.w.data = wdata_i[0]; - axi_req_o.w.strb = be_i[0]; - - if (axi_resp_i.w_ready) - cnt_d = BURST_SIZE - 1; - else - cnt_d = BURST_SIZE; - - case ({axi_resp_i.aw_ready, axi_resp_i.w_ready}) - 2'b11: state_d = WAIT_LAST_W_READY; - 2'b01: state_d = WAIT_LAST_W_READY_AW_READY; - 2'b10: state_d = WAIT_LAST_W_READY; - default:; - endcase end // read end else begin + // only multiple outstanding write transactions are allowed + if (!any_outstanding_aw) begin - axi_req_o.ar_valid = 1'b1; - // load-reserved requires exclusive access - axi_req_o.ar.lock = amo_i == ariane_pkg::AMO_LR; + axi_req_o.ar_valid = 1'b1; + // load-reserved requires exclusive access + axi_req_o.ar.lock = amo_i == ariane_pkg::AMO_LR; - gnt_o = axi_resp_i.ar_ready; - if (type_i != ariane_axi::SINGLE_REQ) begin - assert (amo_i == ariane_pkg::AMO_NONE) - else $fatal("Bursts of atomic operations are not supported"); + gnt_o = axi_resp_i.ar_ready; + if (type_i != ariane_axi::SINGLE_REQ) begin + assert (amo_i == ariane_pkg::AMO_NONE) + else $fatal("Bursts of atomic operations are not supported"); - axi_req_o.ar.len = BURST_SIZE; - cnt_d = BURST_SIZE; - end + axi_req_o.ar.len = BURST_SIZE; + cnt_d = BURST_SIZE; + end + + if (axi_resp_i.ar_ready) begin + state_d = (type_i == ariane_axi::SINGLE_REQ) ? WAIT_R_VALID : WAIT_R_VALID_MULTIPLE; + addr_offset_d = addr_i[ADDR_INDEX-1+3:3]; + end - if (axi_resp_i.ar_ready) begin - state_d = (type_i == ariane_axi::SINGLE_REQ) ? WAIT_R_VALID : WAIT_R_VALID_MULTIPLE; - addr_offset_d = addr_i[ADDR_INDEX-1+3:3]; end end end @@ -216,6 +237,7 @@ module axi_adapter #( if (axi_resp_i.aw_ready) begin gnt_o = 1'b1; state_d = WAIT_B_VALID; + id_d = id_i; amo_d = amo_i; size_d = size_i; end @@ -299,7 +321,7 @@ module axi_adapter #( id_o = axi_resp_i.b.id; // Write is valid - if (axi_resp_i.b_valid) begin + if (axi_resp_i.b_valid && !any_outstanding_aw) begin axi_req_o.b_ready = 1'b1; // some atomics must wait for read data @@ -333,6 +355,13 @@ module axi_adapter #( end end end + // if the request was not an atomic we can possibly issue + // other requests while waiting for the response + end else begin + if ((amo_q == ariane_pkg::AMO_NONE) && (outstanding_aw_cnt_q != MAX_OUTSTANDING_AW)) begin + state_d = IDLE; + outstanding_aw_cnt_d = outstanding_aw_cnt_q + 1; + end end end @@ -398,6 +427,16 @@ module axi_adapter #( default: state_d = IDLE; endcase + + // This process handles B responses when accepting + // multiple outstanding write transactions + if (any_outstanding_aw && axi_resp_i.b_valid) begin + axi_req_o.b_ready = 1'b1; + valid_o = 1'b1; + // Right hand side contains non-registered signal as we want + // to preserve a possible increment from the WAIT_B_VALID state + outstanding_aw_cnt_d = outstanding_aw_cnt_d - 1; + end end // ---------------- @@ -413,6 +452,7 @@ module axi_adapter #( id_q <= '0; amo_q <= ariane_pkg::AMO_NONE; size_q <= '0; + outstanding_aw_cnt_q <= '0; end else begin state_q <= state_d; cnt_q <= cnt_d; @@ -421,6 +461,7 @@ module axi_adapter #( id_q <= id_d; amo_q <= amo_d; size_q <= size_d; + outstanding_aw_cnt_q <= outstanding_aw_cnt_d; end end diff --git a/core/cache_subsystem/cache_ctrl.sv b/core/cache_subsystem/cache_ctrl.sv index 0062ed1cbf..24d98eb725 100644 --- a/core/cache_subsystem/cache_ctrl.sv +++ b/core/cache_subsystem/cache_ctrl.sv @@ -354,9 +354,13 @@ module cache_ctrl import ariane_pkg::*; import std_cache_pkg::*; #( // got a grant so go to valid if (bypass_gnt_i) begin state_d = WAIT_REFILL_VALID; - // if this was a write we still need to give a grant to the store unit - if (mem_req_q.we) + // if this was a write we still need to give a grant to the store unit. + // We can also avoid waiting for the response valid, this signal is + // currently not used by the store unit + if (mem_req_q.we) begin req_port_o.data_gnt = 1'b1; + state_d = IDLE; + end end if (miss_gnt_i && !mem_req_q.we) diff --git a/core/cache_subsystem/miss_handler.sv b/core/cache_subsystem/miss_handler.sv index 0d9c28865d..499191d56b 100644 --- a/core/cache_subsystem/miss_handler.sv +++ b/core/cache_subsystem/miss_handler.sv @@ -547,9 +547,10 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #( // Arbitrate bypass ports // ---------------------- axi_adapter_arbiter #( - .NR_PORTS(NR_BYPASS_PORTS), - .req_t (bypass_req_t), - .rsp_t (bypass_rsp_t) + .NR_PORTS (NR_BYPASS_PORTS), + .MAX_OUTSTANDING_REQ(7), + .req_t (bypass_req_t), + .rsp_t (bypass_rsp_t) ) i_bypass_arbiter ( .clk_i (clk_i), .rst_ni(rst_ni), @@ -574,6 +575,7 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #( .AXI_ADDR_WIDTH ( AXI_ADDR_WIDTH ), .AXI_DATA_WIDTH ( AXI_DATA_WIDTH ), .AXI_ID_WIDTH ( AXI_ID_WIDTH ), + .MAX_OUTSTANDING_AW ( 7 ), .axi_req_t ( axi_req_t ), .axi_rsp_t ( axi_rsp_t ) ) i_bypass_axi_adapter ( @@ -673,6 +675,7 @@ endmodule // module axi_adapter_arbiter #( parameter NR_PORTS = 4, + parameter MAX_OUTSTANDING_REQ = 0, parameter type req_t = std_cache_pkg::bypass_req_t, parameter type rsp_t = std_cache_pkg::bypass_rsp_t )( @@ -686,13 +689,29 @@ module axi_adapter_arbiter #( input rsp_t rsp_i ); + localparam MAX_OUTSTANDING_CNT_WIDTH = $clog2(MAX_OUTSTANDING_REQ + 1) > 0 ? $clog2(MAX_OUTSTANDING_REQ + 1) : 1; + + typedef logic [MAX_OUTSTANDING_CNT_WIDTH-1:0] outstanding_cnt_t; + enum logic { IDLE, SERVING } state_d, state_q; req_t req_d, req_q; logic [NR_PORTS-1:0] sel_d, sel_q; + outstanding_cnt_t outstanding_cnt_d, outstanding_cnt_q; + + logic [NR_PORTS-1:0] req_flat; + logic any_unselected_port_valid; + + generate + for (genvar i = 0; i < NR_PORTS; i++) begin + assign req_flat[i] = req_i[i].req; + end + endgenerate + assign any_unselected_port_valid = |(req_flat & ~(1 << sel_q)); always_comb begin sel_d = sel_q; + outstanding_cnt_d = outstanding_cnt_q; state_d = state_q; req_d = req_q; @@ -701,6 +720,7 @@ module axi_adapter_arbiter #( rsp_o = '0; rsp_o[sel_q].rdata = rsp_i.rdata; + rsp_o[sel_q].valid = rsp_i.valid; case (state_q) @@ -717,15 +737,43 @@ module axi_adapter_arbiter #( req_d = req_i[sel_d]; req_o = req_i[sel_d]; rsp_o[sel_d].gnt = req_i[sel_d].req; + + // Count outstanding transactions, i.e. requests which have been + // granted but response hasn't arrived yet + if (req_o.req && rsp_i.gnt) begin + req_d.req = 1'b0; + outstanding_cnt_d += 1; + end end SERVING: begin + // Count outstanding transactions, i.e. requests which have been + // granted but response hasn't arrived yet + if (req_o.req && rsp_i.gnt) begin + req_d.req = 1'b0; + outstanding_cnt_d += 1; + end if (rsp_i.valid) begin + outstanding_cnt_d -= 1; rsp_o[sel_q].valid = 1'b1; + + if ((outstanding_cnt_d == 0) && (!req_o.req || rsp_i.gnt)) begin state_d = IDLE; end end + // We can accept multiple outstanding transactions from same port. + // To ensure fairness, we allow this only if all other ports are idle + if ((!req_o.req || rsp_i.gnt) && !any_unselected_port_valid && + (outstanding_cnt_d != MAX_OUTSTANDING_REQ)) begin + if (req_i[sel_q].req) begin + req_d = req_i[sel_q]; + rsp_o[sel_q].gnt = 1'b1; + state_d = SERVING; + end + end + end + default : /* default */; endcase end @@ -735,10 +783,12 @@ module axi_adapter_arbiter #( state_q <= IDLE; sel_q <= '0; req_q <= '0; + outstanding_cnt_q <= '0; end else begin state_q <= state_d; sel_q <= sel_d; req_q <= req_d; + outstanding_cnt_q <= outstanding_cnt_d; end end // ------------ @@ -750,7 +800,7 @@ module axi_adapter_arbiter #( // make sure that we eventually get an rvalid after we received a grant assert property (@(posedge clk_i) rsp_i.gnt |-> ##[1:$] rsp_i.valid ) else begin $error("There was a grant without a rvalid"); $stop(); end - // assert that there is no grant without a request + // assert that there is no grant without a request or outstanding transactions assert property (@(negedge clk_i) rsp_i.gnt |-> req_o.req) else begin $error("There was a grant without a request."); $stop(); end // assert that the address does not contain X when request is sent