Skip to content

Commit

Permalink
cache_subsystem: Support multiple outstanding store operations
Browse files Browse the repository at this point in the history
  • Loading branch information
@colluca
colluca committed Sep 11, 2023
1 parent 372a3bc commit 6f836ff
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 63 deletions.
155 changes: 98 additions & 57 deletions core/cache_subsystem/axi_adapter.sv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ module axi_adapter #(
parameter int unsigned AXI_ADDR_WIDTH = 0,
parameter int unsigned AXI_DATA_WIDTH = 0,
parameter int unsigned AXI_ID_WIDTH = 0,
parameter int unsigned MAX_OUTSTANDING_AW = 0,
parameter type axi_req_t = ariane_axi::req_t,
parameter type axi_rsp_t = ariane_axi::resp_t
)(
Expand Down Expand Up @@ -52,6 +53,9 @@ module axi_adapter #(
);
localparam BURST_SIZE = (DATA_WIDTH/AXI_DATA_WIDTH)-1;
localparam ADDR_INDEX = ($clog2(DATA_WIDTH/AXI_DATA_WIDTH) > 0) ? $clog2(DATA_WIDTH/AXI_DATA_WIDTH) : 1;
localparam MAX_OUTSTANDING_AW_CNT_WIDTH = $clog2(MAX_OUTSTANDING_AW + 1) > 0 ? $clog2(MAX_OUTSTANDING_AW + 1) : 1;

typedef logic [MAX_OUTSTANDING_AW_CNT_WIDTH-1:0] outstanding_aw_cnt_t;

enum logic [3:0] {
IDLE, WAIT_B_VALID, WAIT_AW_READY, WAIT_LAST_W_READY, WAIT_LAST_W_READY_AW_READY, WAIT_AW_READY_BURST,
Expand All @@ -68,6 +72,11 @@ module axi_adapter #(
// save the atomic operation and size
ariane_pkg::amo_t amo_d, amo_q;
logic [1:0] size_d, size_q;
// outstanding write transactions counter
outstanding_aw_cnt_t outstanding_aw_cnt_q, outstanding_aw_cnt_d;
logic any_outstanding_aw;

assign any_outstanding_aw = outstanding_aw_cnt_q != '0;

always_comb begin : axi_fsm
// Default assignments
Expand Down Expand Up @@ -131,6 +140,8 @@ module axi_adapter #(
size_d = size_q;
index = '0;

outstanding_aw_cnt_d = outstanding_aw_cnt_q;

case (state_q)

IDLE: begin
Expand All @@ -140,70 +151,80 @@ module axi_adapter #(
// is this a read or write?
// write
if (we_i) begin
// the data is valid
axi_req_o.aw_valid = 1'b1;
axi_req_o.w_valid = 1'b1;
// store-conditional requires exclusive access
axi_req_o.aw.lock = amo_i == ariane_pkg::AMO_SC;
// its a single write
if (type_i == ariane_axi::SINGLE_REQ) begin
// only a single write so the data is already the last one
axi_req_o.w.last = 1'b1;
// single req can be granted here
gnt_o = axi_resp_i.aw_ready & axi_resp_i.w_ready;
case ({axi_resp_i.aw_ready, axi_resp_i.w_ready})
2'b11: state_d = WAIT_B_VALID;
2'b01: state_d = WAIT_AW_READY;
2'b10: state_d = WAIT_LAST_W_READY;
default: state_d = IDLE;
endcase

if (axi_resp_i.aw_ready) begin
amo_d = amo_i;
size_d = size_i;
// multiple outstanding write transactions are only
// allowed if they are guaranteed not to be reordered
// i.e. same ID
if (!any_outstanding_aw || ((id_i == id_q) && (amo_i == ariane_pkg::AMO_NONE))) begin
// the data is valid
axi_req_o.aw_valid = 1'b1;
axi_req_o.w_valid = 1'b1;
// store-conditional requires exclusive access
axi_req_o.aw.lock = amo_i == ariane_pkg::AMO_SC;
// its a single write
if (type_i == ariane_axi::SINGLE_REQ) begin
// only a single write so the data is already the last one
axi_req_o.w.last = 1'b1;
// single req can be granted here
gnt_o = axi_resp_i.aw_ready & axi_resp_i.w_ready;
case ({axi_resp_i.aw_ready, axi_resp_i.w_ready})
2'b11: state_d = WAIT_B_VALID;
2'b01: state_d = WAIT_AW_READY;
2'b10: state_d = WAIT_LAST_W_READY;
default: state_d = IDLE;
endcase

if (axi_resp_i.aw_ready) begin
id_d = id_i;
amo_d = amo_i;
size_d = size_i;
end

// its a request for the whole cache line
end else begin
// bursts of AMOs unsupported
assert (amo_i == ariane_pkg::AMO_NONE)
else $fatal("Bursts of atomic operations are not supported");

axi_req_o.aw.len = BURST_SIZE; // number of bursts to do
axi_req_o.w.data = wdata_i[0];
axi_req_o.w.strb = be_i[0];

if (axi_resp_i.w_ready)
cnt_d = BURST_SIZE - 1;
else
cnt_d = BURST_SIZE;

case ({axi_resp_i.aw_ready, axi_resp_i.w_ready})
2'b11: state_d = WAIT_LAST_W_READY;
2'b01: state_d = WAIT_LAST_W_READY_AW_READY;
2'b10: state_d = WAIT_LAST_W_READY;
default:;
endcase
end

// its a request for the whole cache line
end else begin
// bursts of AMOs unsupported
assert (amo_i == ariane_pkg::AMO_NONE)
else $fatal("Bursts of atomic operations are not supported");

axi_req_o.aw.len = BURST_SIZE; // number of bursts to do
axi_req_o.w.data = wdata_i[0];
axi_req_o.w.strb = be_i[0];

if (axi_resp_i.w_ready)
cnt_d = BURST_SIZE - 1;
else
cnt_d = BURST_SIZE;

case ({axi_resp_i.aw_ready, axi_resp_i.w_ready})
2'b11: state_d = WAIT_LAST_W_READY;
2'b01: state_d = WAIT_LAST_W_READY_AW_READY;
2'b10: state_d = WAIT_LAST_W_READY;
default:;
endcase
end
// read
end else begin
// only multiple outstanding write transactions are allowed
if (!any_outstanding_aw) begin

axi_req_o.ar_valid = 1'b1;
// load-reserved requires exclusive access
axi_req_o.ar.lock = amo_i == ariane_pkg::AMO_LR;
axi_req_o.ar_valid = 1'b1;
// load-reserved requires exclusive access
axi_req_o.ar.lock = amo_i == ariane_pkg::AMO_LR;

gnt_o = axi_resp_i.ar_ready;
if (type_i != ariane_axi::SINGLE_REQ) begin
assert (amo_i == ariane_pkg::AMO_NONE)
else $fatal("Bursts of atomic operations are not supported");
gnt_o = axi_resp_i.ar_ready;
if (type_i != ariane_axi::SINGLE_REQ) begin
assert (amo_i == ariane_pkg::AMO_NONE)
else $fatal("Bursts of atomic operations are not supported");

axi_req_o.ar.len = BURST_SIZE;
cnt_d = BURST_SIZE;
end
axi_req_o.ar.len = BURST_SIZE;
cnt_d = BURST_SIZE;
end

if (axi_resp_i.ar_ready) begin
state_d = (type_i == ariane_axi::SINGLE_REQ) ? WAIT_R_VALID : WAIT_R_VALID_MULTIPLE;
addr_offset_d = addr_i[ADDR_INDEX-1+3:3];
end

if (axi_resp_i.ar_ready) begin
state_d = (type_i == ariane_axi::SINGLE_REQ) ? WAIT_R_VALID : WAIT_R_VALID_MULTIPLE;
addr_offset_d = addr_i[ADDR_INDEX-1+3:3];
end
end
end
Expand All @@ -216,6 +237,7 @@ module axi_adapter #(
if (axi_resp_i.aw_ready) begin
gnt_o = 1'b1;
state_d = WAIT_B_VALID;
id_d = id_i;
amo_d = amo_i;
size_d = size_i;
end
Expand Down Expand Up @@ -299,7 +321,7 @@ module axi_adapter #(
id_o = axi_resp_i.b.id;

// Write is valid
if (axi_resp_i.b_valid) begin
if (axi_resp_i.b_valid && !any_outstanding_aw) begin
axi_req_o.b_ready = 1'b1;

// some atomics must wait for read data
Expand Down Expand Up @@ -333,6 +355,13 @@ module axi_adapter #(
end
end
end
// if the request was not an atomic we can possibly issue
// other requests while waiting for the response
end else begin
if ((amo_q == ariane_pkg::AMO_NONE) && (outstanding_aw_cnt_q != MAX_OUTSTANDING_AW)) begin
state_d = IDLE;
outstanding_aw_cnt_d = outstanding_aw_cnt_q + 1;
end
end
end

Expand Down Expand Up @@ -398,6 +427,16 @@ module axi_adapter #(

default: state_d = IDLE;
endcase

// This process handles B responses when accepting
// multiple outstanding write transactions
if (any_outstanding_aw && axi_resp_i.b_valid) begin
axi_req_o.b_ready = 1'b1;
valid_o = 1'b1;
// Right hand side contains non-registered signal as we want
// to preserve a possible increment from the WAIT_B_VALID state
outstanding_aw_cnt_d = outstanding_aw_cnt_d - 1;
end
end

// ----------------
Expand All @@ -413,6 +452,7 @@ module axi_adapter #(
id_q <= '0;
amo_q <= ariane_pkg::AMO_NONE;
size_q <= '0;
outstanding_aw_cnt_q <= '0;
end else begin
state_q <= state_d;
cnt_q <= cnt_d;
Expand All @@ -421,6 +461,7 @@ module axi_adapter #(
id_q <= id_d;
amo_q <= amo_d;
size_q <= size_d;
outstanding_aw_cnt_q <= outstanding_aw_cnt_d;
end
end

Expand Down
8 changes: 6 additions & 2 deletions core/cache_subsystem/cache_ctrl.sv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,9 +354,13 @@ module cache_ctrl import ariane_pkg::*; import std_cache_pkg::*; #(
// got a grant so go to valid
if (bypass_gnt_i) begin
state_d = WAIT_REFILL_VALID;
// if this was a write we still need to give a grant to the store unit
if (mem_req_q.we)
// if this was a write we still need to give a grant to the store unit.
// We can also avoid waiting for the response valid, this signal is
// currently not used by the store unit
if (mem_req_q.we) begin
req_port_o.data_gnt = 1'b1;
state_d = IDLE;
end
end

if (miss_gnt_i && !mem_req_q.we)
Expand Down
58 changes: 54 additions & 4 deletions core/cache_subsystem/miss_handler.sv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -547,9 +547,10 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
// Arbitrate bypass ports
// ----------------------
axi_adapter_arbiter #(
.NR_PORTS(NR_BYPASS_PORTS),
.req_t (bypass_req_t),
.rsp_t (bypass_rsp_t)
.NR_PORTS (NR_BYPASS_PORTS),
.MAX_OUTSTANDING_REQ(7),
.req_t (bypass_req_t),
.rsp_t (bypass_rsp_t)
) i_bypass_arbiter (
.clk_i (clk_i),
.rst_ni(rst_ni),
Expand All @@ -574,6 +575,7 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
.AXI_ADDR_WIDTH ( AXI_ADDR_WIDTH ),
.AXI_DATA_WIDTH ( AXI_DATA_WIDTH ),
.AXI_ID_WIDTH ( AXI_ID_WIDTH ),
.MAX_OUTSTANDING_AW ( 7 ),
.axi_req_t ( axi_req_t ),
.axi_rsp_t ( axi_rsp_t )
) i_bypass_axi_adapter (
Expand Down Expand Up @@ -673,6 +675,7 @@ endmodule
//
module axi_adapter_arbiter #(
parameter NR_PORTS = 4,
parameter MAX_OUTSTANDING_REQ = 0,
parameter type req_t = std_cache_pkg::bypass_req_t,
parameter type rsp_t = std_cache_pkg::bypass_rsp_t
)(
Expand All @@ -686,13 +689,29 @@ module axi_adapter_arbiter #(
input rsp_t rsp_i
);

localparam MAX_OUTSTANDING_CNT_WIDTH = $clog2(MAX_OUTSTANDING_REQ + 1) > 0 ? $clog2(MAX_OUTSTANDING_REQ + 1) : 1;

typedef logic [MAX_OUTSTANDING_CNT_WIDTH-1:0] outstanding_cnt_t;

enum logic { IDLE, SERVING } state_d, state_q;

req_t req_d, req_q;
logic [NR_PORTS-1:0] sel_d, sel_q;
outstanding_cnt_t outstanding_cnt_d, outstanding_cnt_q;

logic [NR_PORTS-1:0] req_flat;
logic any_unselected_port_valid;

generate
for (genvar i = 0; i < NR_PORTS; i++) begin
assign req_flat[i] = req_i[i].req;
end
endgenerate
assign any_unselected_port_valid = |(req_flat & ~(1 << sel_q));

always_comb begin
sel_d = sel_q;
outstanding_cnt_d = outstanding_cnt_q;

state_d = state_q;
req_d = req_q;
Expand All @@ -701,6 +720,7 @@ module axi_adapter_arbiter #(

rsp_o = '0;
rsp_o[sel_q].rdata = rsp_i.rdata;
rsp_o[sel_q].valid = rsp_i.valid;

case (state_q)

Expand All @@ -717,15 +737,43 @@ module axi_adapter_arbiter #(
req_d = req_i[sel_d];
req_o = req_i[sel_d];
rsp_o[sel_d].gnt = req_i[sel_d].req;

// Count outstanding transactions, i.e. requests which have been
// granted but response hasn't arrived yet
if (req_o.req && rsp_i.gnt) begin
req_d.req = 1'b0;
outstanding_cnt_d += 1;
end
end

SERVING: begin
// Count outstanding transactions, i.e. requests which have been
// granted but response hasn't arrived yet
if (req_o.req && rsp_i.gnt) begin
req_d.req = 1'b0;
outstanding_cnt_d += 1;
end
if (rsp_i.valid) begin
outstanding_cnt_d -= 1;
rsp_o[sel_q].valid = 1'b1;

if ((outstanding_cnt_d == 0) && (!req_o.req || rsp_i.gnt)) begin
state_d = IDLE;
end
end

// We can accept multiple outstanding transactions from same port.
// To ensure fairness, we allow this only if all other ports are idle
if ((!req_o.req || rsp_i.gnt) && !any_unselected_port_valid &&
(outstanding_cnt_d != MAX_OUTSTANDING_REQ)) begin
if (req_i[sel_q].req) begin
req_d = req_i[sel_q];
rsp_o[sel_q].gnt = 1'b1;
state_d = SERVING;
end
end
end

default : /* default */;
endcase
end
Expand All @@ -735,10 +783,12 @@ module axi_adapter_arbiter #(
state_q <= IDLE;
sel_q <= '0;
req_q <= '0;
outstanding_cnt_q <= '0;
end else begin
state_q <= state_d;
sel_q <= sel_d;
req_q <= req_d;
outstanding_cnt_q <= outstanding_cnt_d;
end
end
// ------------
Expand All @@ -750,7 +800,7 @@ module axi_adapter_arbiter #(
// make sure that we eventually get an rvalid after we received a grant
assert property (@(posedge clk_i) rsp_i.gnt |-> ##[1:$] rsp_i.valid )
else begin $error("There was a grant without a rvalid"); $stop(); end
// assert that there is no grant without a request
// assert that there is no grant without a request or outstanding transactions
assert property (@(negedge clk_i) rsp_i.gnt |-> req_o.req)
else begin $error("There was a grant without a request."); $stop(); end
// assert that the address does not contain X when request is sent
Expand Down

0 comments on commit 6f836ff

Please sign in to comment.