Improve Xilinx memory code generation and DDR5 training

This commit adds Xilinx specific Verilog code generation
for the Memory objects. New code is compliant with UG901.
This commit also fixes issue with CS-CA phase detection,
which caused incorrect QCS, QCK and QCA delay values to be used.

Signed-off-by: Maciej Dudek <[email protected]>

litex/build/xilinx/common.py

@@ -14,6 +14,7 @@
 from migen.fhdl.structure import *
 from migen.fhdl.specials import Instance, Tristate
 from migen.fhdl.module import Module
+from migen.fhdl.verilog import _printexpr as verilog_printexpr
 from migen.genlib.cdc import *
 from migen.genlib.resetsync import AsyncResetSynchronizer
 
@@ -187,6 +188,165 @@ class XilinxDDRTristate:
     def lower(dr):
         return XilinxDDRTristateImpl(dr.io, dr.o1, dr.o2, dr.oe1, dr.oe2, dr.i1, dr.i2, dr.clk)
 
+# Common Memory ------------------------------------------------------------------------------------
+
+class XilinxMemoryImpl:
+    @staticmethod
+    def emit_verilog(memory, namespace, add_data_file):
+        # Helpers.
+        # --------
+
+        def _get_name(e):
+            if isinstance(e, Memory):
+                return namespace.get_name(e)
+            else:
+                return verilog_printexpr(namespace, e)[0]
+
+        # Parameters.
+        # -----------
+        r         = ""
+        adr_regs  = {}
+        data_regs = {}
+
+        # Ports Transformations.
+        # ----------------------
+
+        # Set Port Mode to Read-First when several Ports with different Clocks.
+        # FIXME: Verify behaviour with the different FPGA toolchains, try to avoid it.
+        clocks = [port.clock for port in memory.ports]
+        if clocks.count(clocks[0]) != len(clocks):
+            for port in memory.ports:
+                port.mode = READ_FIRST
+
+        # Set Port Granularity when 0.
+        for port in memory.ports:
+            if port.we_granularity == 0:
+                port.we_granularity = memory.width
+
+        # Memory Description.
+        # -------------------
+        r += "//" + "-"*78 + "\n"
+        r += f"// Memory {_get_name(memory)}: {memory.depth}-words x {memory.width}-bit\n"
+        r += "//" + "-"*78 + "\n"
+        for n, port in enumerate(memory.ports):
+            r += f"// Port {n} | "
+            if port.async_read:
+                r += "Read: Async | "
+            else:
+                r += "Read: Sync  | "
+            if port.we is None:
+                r += "Write: ---- | "
+            else:
+                r += "Write: Sync | "
+                r += "Mode: "
+                if port.mode == WRITE_FIRST:
+                    r += "Write-First | "
+                elif port.mode == READ_FIRST:
+                    r += "Read-First  | "
+                elif port.mode == NO_CHANGE:
+                    r += "No-Change | "
+                r += f"Write-Granularity: {port.we_granularity} "
+            r += "\n"
+
+        # Memory Logic Declaration/Initialization.
+        # ----------------------------------------
+        r += f"reg [{memory.width-1}:0] {_get_name(memory)}[0:{memory.depth-1}];\n"
+        if memory.init is not None:
+            content = ""
+            formatter = f"{{:0{int(memory.width/4)}x}}\n"
+            for d in memory.init:
+                content += formatter.format(d)
+            memory_filename = add_data_file(f"{memory.LiteX_name}_{_get_name(memory)}.init", content)
+
+            r += "initial begin\n"
+            r += f"\t$readmemh(\"{memory_filename}\", {_get_name(memory)});\n"
+            r += "end\n"
+
+        # Port Intermediate Signals.
+        # --------------------------
+        for n, port in enumerate(memory.ports):
+            # No Intermediate Signal for Async Read.
+            if port.async_read:
+                continue
+
+            # Create Address Register in Write-First mode.
+            if port.mode in [WRITE_FIRST]:
+                adr_regs[n] = Signal(name_override=f"{_get_name(memory)}_adr{n}")
+                r += f"reg [{bits_for(memory.depth-1)-1}:0] {_get_name(adr_regs[n])};\n"
+
+            # Create Data Register in Read-First/No Change mode.
+            if port.mode in [READ_FIRST, NO_CHANGE]:
+                data_regs[n] = Signal(name_override=f"{_get_name(memory)}_dat{n}")
+                r += f"reg [{memory.width-1}:0] {_get_name(data_regs[n])};\n"
+
+        # Ports Write/Read Logic.
+        # -----------------------
+        for n, port in enumerate(memory.ports):
+            loop_var_name = f"{_get_name(memory)}_{n}_loop_var"
+            # Declare loop variable before always block
+            if port.we is not None and memory.width != port.we_granularity:
+                r += f"integer {loop_var_name};\n"
+            r += f"always @(posedge {_get_name(port.clock)}) begin\n"
+            # Write Logic.
+            if port.we is not None:
+                # Create for loop with we.
+                preamble = ""
+                postamble = ""
+                if memory.width != port.we_granularity:
+                    preamble += f"for({loop_var_name} = 0; "
+                    preamble += f"{loop_var_name} < {memory.width//port.we_granularity}; "
+                    preamble += f"{loop_var_name}={loop_var_name}+1) begin\n"
+                    postamble = "end\n"
+                r += preamble
+                wbit = f"[{loop_var_name}]" if memory.width != port.we_granularity else ""
+                r += f"\tif ({_get_name(port.we)}{wbit}) begin\n"
+                dslc = f"[{loop_var_name}*{port.we_granularity}+: {port.we_granularity}]" if (memory.width != port.we_granularity) else ""
+                r += f"\t\t{_get_name(memory)}[{_get_name(port.adr)}]{dslc} <= {_get_name(port.dat_w)}{dslc};\n"
+                r += "\tend\n"
+                r += postamble
+
+            # Read Logic.
+            if not port.async_read:
+                # In Write-First mode, Read from Address Register.
+                if port.mode in [WRITE_FIRST]:
+                    rd = f"\t{_get_name(adr_regs[n])} <= {_get_name(port.adr)};\n"
+
+                # In Read-First/No Change mode:
+                if port.mode in [READ_FIRST, NO_CHANGE]:
+                    rd = ""
+                    # Only Read in No-Change mode when no Write.
+                    if port.mode == NO_CHANGE:
+                        rd += f"\tif (!{_get_name(port.we)})\n\t"
+                    # Read-First/No-Change Read logic.
+                    rd += f"\t{_get_name(data_regs[n])} <= {_get_name(memory)}[{_get_name(port.adr)}];\n"
+
+                # Add Read-Enable Logic.
+                if port.re is None:
+                    r += rd
+                else:
+                    r += f"\tif ({_get_name(port.re)})\n"
+                    r += "\t" + rd.replace("\n\t", "\n\t\t")
+            r += "end\n"
+
+        # Ports Read Mapping.
+        # -------------------
+        for n, port in enumerate(memory.ports):
+            # Direct (Asynchronous) Read on Async-Read mode.
+            if port.async_read:
+                r += f"assign {_get_name(port.dat_r)} = {_get_name(memory)}[{_get_name(port.adr)}];\n"
+                continue
+
+            # Write-First mode: Do Read through Address Register.
+            if port.mode in [WRITE_FIRST]:
+                r += f"assign {_get_name(port.dat_r)} = {_get_name(memory)}[{_get_name(adr_regs[n])}];\n"
+
+            # Read-First/No-Change mode: Data already Read on Data Register.
+            if port.mode in [READ_FIRST, NO_CHANGE]:
+                 r += f"assign {_get_name(port.dat_r)} = {_get_name(data_regs[n])};\n"
+        r += "\n\n"
+
+        return r
+
 # Common Special Overrides -------------------------------------------------------------------------
 
 xilinx_special_overrides = {
@@ -196,6 +356,7 @@ def lower(dr):
     DifferentialOutput:     XilinxDifferentialOutput,
     SDRTristate:            XilinxSDRTristate,
     DDRTristate:            XilinxDDRTristate,
+    Memory:                 XilinxMemoryImpl,
 }
 
 # Spartan6 DDROutput -------------------------------------------------------------------------------

litex/soc/software/liblitedram/ddr5_training.c

@@ -191,7 +191,7 @@ static void CS_training(training_ctx_t *const ctx, int32_t channel, uint8_t *suc
         find_eye_in_helper_arr(&left_side, &right_side, ctx->max_delay_taps);
         right_side -= ctx->max_delay_taps;
         left_side -= ctx->max_delay_taps;
-        if (right_side < 0) {
+        if (left_side < 0) {
             right_side = ctx->cs.delays[channel][_rank][0] + ctx->max_delay_taps;
             left_side = ctx->cs.delays[channel][_rank][1] + ctx->max_delay_taps;