docs/images/DIAG01-STAGE15-add-attention-output-and-current-tensor.drawio.svg

<svg host="65bd71144e" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="1657px" height="971px" viewBox="-0.5 -0.5 1657 971" content="&lt;mxfile&gt;&lt;diagram id=&quot;FOsQcooNq0DZRFAvDWz8&quot; name=&quot;Page-1&quot;&gt;7V1rc5u6Fv01mWk/2IMQz492Hqed256503bm3H4kBttMMfhg3CT99VcCxEMSDmBhE6Kk0xgZBOy9tCXttRA38Hb3/Ffs7LdfI9cLblTFfb6BdzeqqmqKif7gkpesZGaoWlayiX03KwNlwXf/j5cXKnnp0Xe9Q23HJIqCxN/XC1dRGHqrpFbmxHH0VN9tHQX1s+6djccUfF85AVv6j+8m26zUUs2y/JPnb7bkzMCws292Dtk5v5PD1nGjp0oRvL+Bt3EUJdmn3fOtF2DrEbtkxz00fFtcWOyFSZsDVEhM/9sJjvnt5ZeWvJD7jaNj6Hr4EOUGLp+2fuJ93zsr/O0TcjEq2ya7AG0B9PGQxNEv7zYKojg9GhoL/Iu+YS8uv97fXpx4z5Wi/GL/8qKdl8QvaJf82xkwc8vl2AGGkhc8lZ4Ahp4XbitusMmhTu7+TVF9aSH0ITdSg4UVVbC98M37CFuLwN+EqOwxSpJoh75Y+0FQMeODjX95Bl7CW/O+ND2BpCrK5AZtco1ncsgxuWULMLlq2KJtTpn2/gH/4vIoTEh5GIXo4GUYfXEevUZwD217lbK9aais7XWTY3tTt0TAHXJMbwQJvuO9E9Z8YPx7xHFriW4umTkZnBdojzi7qGW5B/q0wX+/BM7OQXvAObKD8vcxOeDrDV30/xLFc7z1OVx7yILIkapyH2585JT8/Ojas0vI6mIQgS/jpMuaGlgVKAgQef+DAmW+nZ+C13h3vuviL5f57d8F3jqhgIVOhX4AVLmoKXGM698EzuFwAtPDtG/Atm9d4UFMSEA1GIT9iJ3wsI7inRejL7IBBEZS5vfHuAaf2s6Gs8NmCR8P+/cAiWFCjGrPLcAgAFpDIQAABgFCw/tycWvdwStEb5uyrKVZnOCtcuxqkDB/nmH1ocd2JwxI97H2g3ZvCDKsXjesClSdNaxmcAyrGkIQyw5I6jFrGUSrX+jvhy/OS1oAPmZ9IR2y8h0nHqeG6rqYBgbYBjZY3FIV0ieeal8bZJU948t8cIlt++Ds/ADfwCcv+O1hl9SdBwDrhAcF/1I4yOsM8HB16ax+bVJ/UN/uIz9MvPj+N7L6IfdUBTTtnVLMrp1HcqvKSWdpplpvtAq0GWepvJGsbghxVqdeBrToZXo6b53+sI1YUSxHUXjNXlHQCF/p5VvWoRXYvt7QXnfNXKn+ACGeajHdO2ydPf6Yww/ZzImJ0bGlEDgTB00W4twOqygInP3Bz8Ca7rH1AxeF5+iYkHrIVm/XFqkOTrvkBgDBjbWTb0m7nENds6BlKrZhmhqwa60UzqFR8zBk26wB5qqtQIQGSzE0XeXAREy45c1FT+HiG06tLbdR7P/BcAhyz9NYOTz5u8AJvU+e41JFyyhNUuKiJNrnn7KOMf1IkjXpRj7PzfFH482No/0PJ954ZBcORlKHpzbSl+gfstqtMtdvdHRnt2gblNvoH949RugLEbIQ1HG1nnNInryDEAjT44P0/tnY9JD+DAJltTWUu6GRQPxMNPISp/kMlZsWeYxi14tnq8wwi/TQ+MNsVi3/mNpCwT6ZHVIn4f2Aun/m5k9eTcWkFT3l946r0hSl2yl+eOGBXG9D5mVbKX9sTMeQlomcnPhO8A0PHcJNGrwrQOD1ufWBBr7+CGFiHaSJ8y0atnohvzl1a7I8iLZqReqJARo1mBbbqLitBvZpNcC0u7WTvLbSi92rcwJk7dBJUJBFt3ZgWl9xpW0bJDvbXT4EkZMAQ4KxBkaSkB0VGp8p6DxVmTWR4GxTm3BssklOOXQZbOgyzkHKSx1q3cYsUMzMymRg+OHg/XvE/MoXL9ygC0LeUpU7f/dRxsw2HXjB1gqNmkZrUL3W6Z7CTY8+nK5OeJy0WIDigtt06GobEpXtevJrw7JjZ94fpW1qEwxSqEEGhZ678UhyneR7cmDmMMPGpTLkFXx6z37yPwwY1EVmWz9z+ODPd8/VDdKrNzjQ9dbOMeDSB1QHWWNrWiDSONHleqG7wMoqfMM48e+v0kJk1cpt4c2f5I7xRnlj6Ra5MxZ8uUcP0TFeeaTTtPMBf0KGJllpPi/HLjmJ0dgLnMT/7dXOcAKB/8UNucS2Tk5f0OYkwU3qyK42P6zEHFOTQbrlgsYwtXpN2R0yNfVDL5u3mM/nDKK7pcLrmey1tfJWKx4CHy1d0xUWR0tS3uj6DglNimgAlsHqGbjZbCgiTQnJ8G2yBjYsin7VIScrPKCB2fn8NeJv3+g2/rjdKv5CQAZrtfhbxJeh4y8TNW2D4glbx1+NopFtS6nRWvpw0dhowUpWfJRj23UO25MwroAYtAMx6ALhOg51HmYhmsdqD0yba9e1QxKx69Aipa9CqyZcY8MQKTsTgTOgUIHQVntCkFG0WCZVk1DQtSDSzgNdy8hZA51yddCpXNCpIwMd1fui7rAv6ACkqwKDoQ4ANgX6FfV3/izNXarKdy9YzxZJgnzkR5jN+hq5R4zGQjZV+ZICq9RL9dPNKTarmxtOLwVVlqG5851N7OxQ4SO+0IXr40MWwT5Vzd0tvvzn87fF31kCCP1qTa5vFCkSpRWbxnkth1I4mOeNumyyUQZSwRCk0juCtJAAUE7lyrchT2QqZDRuWBbbd8jR+M2lR+OKCTm9V6HIuvRoHLfWfr0SO663hxsKFdmiyU7WmWyIYV8yG1Ik6d6PgU3rsgYeRzZk/HF02Gx0kfegstH6heIvoHPIwJ7blR/qYcDW0Ziu16RRKy4am2aL51Wknr4pCwvJUymX0NObZgtFy3uQ05egHaWa3jRZyQfjJymmP9ezl9TSC0EFq7M4jQqpR3u7UvoyBohW0tNzvH5YtDlYnLSQnn5Knauqf5aq+rGo6stweb4g71ST6SHIo6sTq3UqprJSUz8WTX0XLHZU4fWHZpvaRCOzBXEuRzBvTlHfZazyUgfa5QX1JsH9K3r6en8vxfVXFNeXceOdaetNixV9SGn9OKT1XUA5YWW9BdlwyoBScst9+/D+5HKR7K1zG2RyMDS3YYG60h7dk1mvQxwHYUEeB5FNxF3/95n5iKa0Q7mkXWXSn56uYdqPTZ6vS1kFN4ldKw/HRU7wKjQtsXfw/1Tic32wedcAq0b88fDGQqts4GezH8ocFhLJl1pFfcFGdonW64N3vrCAM3OeFu/N8E+qwYrJBnwKwGANfMK6Taq6inWrArx6+G2lKqvKTyN0Ej/BhtEEWfuchY+hIsbgbGT8nqQrZeOQt3BdfIaKfnOX6TdVJTom+2PaxNKVXVfHODUE6k6ylCjttfMlfQ3zif6KvmW1QVVVoQMo+mZAp58s40h2DPJUguiWBRUiaJE0fIuOiFnWDmgtw6AIGr6U/b13Hr4C21ES8ej6Wqy0L5n4s337xpa1Q/fWQvUkM9kT4eIrYWCMy9qhy+PpuibNxjcta1cdy5IhrGTkx8HIV6LmO1vnDt05q+iTnDwXjJfi5DuhccLr3CE7SF3hFFn5TqOWq69zh66WlRTKde7GRsVXosU74+KhwmEHJBk/DjK+EywnzMYjkPKE2RQsq3Q86uq20SYKneC+LJXs/NDsfMnEVen5MjM8OD2vUvS8Qb/1sYGef70mU6GI/obHFnvB2+YRiJLpF8L0V4LH1Kl+BKQWUmTJlzXJBjSt5eKBYviybu+SnTJfZrdnpq/Cl9ldX/fDyRu8V76sg2/fHF/GWbLlNC5kjukt82V2rxfaXIwv46xvM3G+7JynV+vHfvgc7o+JzHVciUKzezHRU6DQbJbklhRap8TbVdE4aQrN7rpgixzevA0KrcNAZgQUms15E4+k0EZHodm9lnaZAoVmczheSaGNhELrAMspU2jFCwLez1Ng0L7gu2CQgdu8P0M+MtxzdHTW22E4pGTxIPHFOUm95bKlLWrShuMkQfE6UclJiucky2AxeU4SKG1e8SLj4uXjYkFA1sQaRbS8vFiDIOXswFgMBQYJjDzSRwZGUYGxKTUxwcB4erl8ERq2IsRVA1wl3gkLcbcm1CtSgeqrsYZZQZ/3pq8i+PMD/nlRcZjV89GdOS+VHfJW1hg0ZyatWTHBydjIHEDenVwiN7sG/uH0pEozTHNuKeVi/tQLCYUGWpbPzJZ2eL4hCzhsb8jSDQemLZ0zkwWPDvBOpTI4ihjjftEvWds6r9II4DNeEWITfcaFJsktKAUpUGt0FgBzq6W/hGjUihervnuNWonccWrUgNJCIi81amf79q1p1ADgactP4UKSuG9Yo1aGgVFq1ADgJcknrVHDY1SuLk0u4TAa/VkZJN+b/gyQN3VJ/dlo9Gdd0Dhl/RkAXR+7kEOXN6E/6zJIub7+DAA24S/1Z6PTn5XR4r3pzwBgE6VSfzYS/VkXWE5af8YZT0rWXyQlxoKPt0JDkW28CL316kucVY0sxH/u66BVaFEvlhb3KgaEXlOi9/roLXFKode6EnqhOPQCtaB2BgFwCyJHIvoKS+ZAMnGpL5lDVAgXV2GZQJQ81aQpY5HiADIhlCqsIVRYoGndsampsJAd2AQo48YJv7xF1Ul7f/XlLYYAPhLZm82m/IyOuNrYS0+I/4i1P+9BFNYzTQ+zUE6Io8RJl+JGXYd56pGWSsNUNUHe04A1J+ufEgcCzZxzRDwGUObk+YBa0hbAOcH8eZ40WzzwQsYPBx/PQvM+sr8H6e695j/K3o1+qfu5KB+isSlUYwOWOSdWqvhK4+mt9M4+QptxhDuuMhgi226/Ri7ugO7/Dw==&lt;/diagram&gt;&lt;/mxfile&gt;">
    <defs/>
    <g>
        <rect x="2" y="0" width="1650" height="970" fill="rgb(255, 255, 255)" stroke="#6a6a6a" pointer-events="all"/>
        <rect x="12" y="40" width="1630" height="890" fill="#f9f9f9" stroke="#b3c7e1" stroke-width="2" pointer-events="all"/>
        <rect x="52" y="162" width="1570" height="758" fill="#efefef" stroke="#b3c7e1" stroke-width="2" pointer-events="all"/>
        <rect x="12" y="10" width="500" height="70" rx="10.5" ry="10.5" fill="#f9f9f9" stroke="#b3c7e1" stroke-width="2" pointer-events="all"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 498px; height: 1px; padding-top: 45px; margin-left: 14px;">
                        <div data-drawio-colors="color: #333132; " style="box-sizing: border-box; font-size: 0px; text-align: left;">
                            <div style="display: inline-block; font-size: 24px; font-family: Helvetica; color: rgb(51, 49, 50); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">
                                <span style="text-align: right;">
                                    Llama 3.1 Nuts and Bolts Inference Engine
                                </span>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="14" y="52" fill="#333132" font-family="Helvetica" font-size="24px" font-weight="bold">
                    Llama 3.1 Nuts and Bolts Inference Engine
                </text>
            </switch>
        </g>
        <rect x="52" y="129.81" width="380" height="70" rx="10.5" ry="10.5" fill="#f9f9f9" stroke="#b3c7e1" stroke-width="2" pointer-events="all"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 1px; height: 1px; padding-top: 165px; margin-left: 54px;">
                        <div data-drawio-colors="color: #333132; " style="box-sizing: border-box; font-size: 0px; text-align: left;">
                            <div style="display: inline-block; font-size: 24px; font-family: Helvetica; color: rgb(51, 49, 50); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: nowrap;">
                                Transformer Model:
                                <br/>
                                LlamaTransformer
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="54" y="172" fill="#333132" font-family="Helvetica" font-size="24px" font-weight="bold">
                    Transformer Model:...
                </text>
            </switch>
        </g>
        <rect x="82" y="248" width="1520" height="662" fill="#bac8d3" stroke="#b3c7e1" stroke-width="2" pointer-events="all"/>
        <rect x="122" y="525" width="1460" height="260" fill="#e9f4e6" stroke="#b3c7e1" stroke-width="2" pointer-events="all"/>
        <rect x="82" y="218" width="380" height="70" rx="10.5" ry="10.5" fill="#f9f9f9" stroke="#b3c7e1" stroke-width="2" pointer-events="all"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 378px; height: 1px; padding-top: 253px; margin-left: 84px;">
                        <div data-drawio-colors="color: #333132; " style="box-sizing: border-box; font-size: 0px; text-align: left;">
                            <div style="display: inline-block; font-size: 24px; font-family: Helvetica; color: rgb(51, 49, 50); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">
                                Transformer Block (Layer 1): LlamaTransformerBlock
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="84" y="260" fill="#333132" font-family="Helvetica" font-size="24px" font-weight="bold">
                    Transformer Block (Layer 1): Lla...
                </text>
            </switch>
        </g>
        <rect x="644" y="439" width="270" height="56" fill="none" stroke="none" pointer-events="all"/>
        <rect x="644" y="439" width="270" height="56" rx="8.4" ry="8.4" fill="#008a00" stroke="#005700" pointer-events="all"/>
        <path d="M 648.35 442.36 M 827.35 442.36 M 827.35 461.36 M 648.35 461.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 449px; margin-left: 650px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 15px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                <b style="border-color: var(--border-color); font-size: 12px;">
                                    <span style="font-weight: 400; font-size: 12px;">
                                        Tensor:
                                    </span>
                                    h
                                </b>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="650" y="461" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    Tensor: h
                </text>
            </switch>
        </g>
        <path d="M 827.35 442.36 M 909.35 442.36 M 909.35 461.36 M 827.35 461.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 449px; margin-left: 827px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 15px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                BFloat16
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="907" y="461" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    BFloat16
                </text>
            </switch>
        </g>
        <path d="M 648.35 461.36 M 827.35 461.36 M 827.35 492.36 M 648.35 492.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 489px; margin-left: 650px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (sequenceLength, Dim)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="650" y="489" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    (sequenceLength, Dim)
                </text>
            </switch>
        </g>
        <path d="M 827.35 461.36 M 909.35 461.36 M 909.35 492.36 M 827.35 492.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 489px; margin-left: 827px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (22, 4096)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="907" y="489" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    (22, 4096)
                </text>
            </switch>
        </g>
        <path d="M 779 180 L 779 128.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 779 122.24 L 783 130.24 L 779 128.24 L 775 130.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <rect x="644" y="260" width="270" height="30" rx="4.5" ry="4.5" fill="#f8cecc" stroke="#b85450" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 268px; height: 1px; padding-top: 275px; margin-left: 645px;">
                        <div data-drawio-colors="color: #B85450; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(184, 84, 80); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                ...
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="779" y="279" fill="#B85450" font-family="Helvetica" font-size="12px" text-anchor="middle">
                    ...
                </text>
            </switch>
        </g>
        <rect x="852" y="933" width="270" height="30" rx="4.5" ry="4.5" fill="#f8cecc" stroke="#b85450" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 268px; height: 1px; padding-top: 948px; margin-left: 853px;">
                        <div data-drawio-colors="color: #B85450; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(184, 84, 80); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                ...
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="987" y="952" fill="#B85450" font-family="Helvetica" font-size="12px" text-anchor="middle">
                    ...
                </text>
            </switch>
        </g>
        <path d="M 779 311.81 L 779 298.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 779 292.24 L 783 300.24 L 779 298.24 L 775 300.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 2 300 L 1652 300" fill="none" stroke="#36454f" stroke-width="5" stroke-miterlimit="10" stroke-dasharray="15 15" pointer-events="none"/>
        <path d="M 2 910 L 1652 910" fill="none" stroke="#36454f" stroke-width="5" stroke-miterlimit="10" stroke-dasharray="15 15" pointer-events="none"/>
        <rect x="122" y="495" width="380" height="70" rx="10.5" ry="10.5" fill="#f9f9f9" stroke="#b3c7e1" stroke-width="2" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 378px; height: 1px; padding-top: 530px; margin-left: 124px;">
                        <div data-drawio-colors="color: #333132; " style="box-sizing: border-box; font-size: 0px; text-align: left;">
                            <div style="display: inline-block; font-size: 24px; font-family: Helvetica; color: rgb(51, 49, 50); line-height: 1.2; pointer-events: none; font-weight: bold; white-space: normal; overflow-wrap: normal;">
                                Multi-Head Self-Attention Module: LlamaAttention
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="124" y="537" fill="#333132" font-family="Helvetica" font-size="24px" font-weight="bold">
                    Multi-Head Self-Attention Module...
                </text>
            </switch>
        </g>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-end; width: 318px; height: 1px; padding-top: 25px; margin-left: 1322px;">
                        <div data-drawio-colors="color: #6A6A6A; " style="box-sizing: border-box; font-size: 0px; text-align: right;">
                            <div style="display: inline-block; font-size: 16px; font-family: Helvetica; color: rgb(106, 106, 106); line-height: 1.2; pointer-events: none; font-weight: bold; font-style: italic; white-space: normal; overflow-wrap: normal;">
                                Diagram by Adil Alper DALKIRAN, 2024
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="1640" y="30" fill="#6A6A6A" font-family="Helvetica" font-size="16px" text-anchor="end" font-weight="bold" font-style="italic">
                    Diagram by Adil Alper DALKIRAN, 2024
                </text>
            </switch>
        </g>
        <path d="M 779 545 L 779 503.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 779 497.24 L 783 505.24 L 779 503.24 L 775 505.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <rect x="644" y="90" width="270" height="30" rx="4.5" ry="4.5" fill="#f8cecc" stroke="#b85450" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 268px; height: 1px; padding-top: 105px; margin-left: 645px;">
                        <div data-drawio-colors="color: #B85450; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(184, 84, 80); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                ...
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="779" y="109" fill="#B85450" font-family="Helvetica" font-size="12px" text-anchor="middle">
                    ...
                </text>
            </switch>
        </g>
        <rect x="644" y="180" width="270" height="30" rx="4.5" ry="4.5" fill="#f8cecc" stroke="#b85450" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 268px; height: 1px; padding-top: 195px; margin-left: 645px;">
                        <div data-drawio-colors="color: #B85450; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(184, 84, 80); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                ...
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="779" y="199" fill="#B85450" font-family="Helvetica" font-size="12px" text-anchor="middle">
                    ...
                </text>
            </switch>
        </g>
        <path d="M 779 260 L 779 218.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 779 212.24 L 783 220.24 L 779 218.24 L 775 220.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <rect x="852" y="712" width="270" height="56" rx="8.4" ry="8.4" fill="#008a00" stroke="#005700" pointer-events="none"/>
        <path d="M 856.35 715.36 M 1035.35 715.36 M 1035.35 735.36 M 856.35 735.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 722px; margin-left: 858px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 16px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                <b style="border-color: var(--border-color); font-size: 12px;">
                                    <span style="font-weight: 400; font-size: 12px;">
                                        Tensor:
                                    </span>
                                    x
                                </b>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="858" y="734" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    Tensor: x
                </text>
            </switch>
        </g>
        <path d="M 1035.35 715.36 M 1117.35 715.36 M 1117.35 735.36 M 1035.35 735.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 722px; margin-left: 1035px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 16px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                BFloat16
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="1115" y="734" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    BFloat16
                </text>
            </switch>
        </g>
        <path d="M 856.35 735.36 M 1035.35 735.36 M 1035.35 766.36 M 856.35 766.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 763px; margin-left: 858px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (sequenceLength, Dim)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="858" y="763" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    (sequenceLength, Dim)
                </text>
            </switch>
        </g>
        <path d="M 1035.35 735.36 M 1117.35 735.36 M 1117.35 766.36 M 1035.35 766.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 763px; margin-left: 1035px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (22, 4096)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="1115" y="763" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    (22, 4096)
                </text>
            </switch>
        </g>
        <path d="M 987 712 L 987 703.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 987 697.24 L 991 705.24 L 987 703.24 L 983 705.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 1px; height: 1px; padding-top: 701px; margin-left: 987px;">
                        <div data-drawio-colors="color: #6A6A6A; background-color: rgb(255, 255, 255); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 11px; font-family: Helvetica; color: rgb(106, 106, 106); line-height: 1.2; pointer-events: none; background-color: rgb(255, 255, 255); white-space: nowrap;">
                                <div style="border-color: var(--border-color);">
                                    <br/>
                                </div>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="987" y="704" fill="#6A6A6A" font-family="Helvetica" font-size="11px" text-anchor="middle">
                </text>
            </switch>
        </g>
        <rect x="852" y="665" width="270" height="30" rx="4.5" ry="4.5" fill="#f8cecc" stroke="#b85450" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 268px; height: 1px; padding-top: 680px; margin-left: 853px;">
                        <div data-drawio-colors="color: #B85450; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(184, 84, 80); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                ...
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="987" y="684" fill="#B85450" font-family="Helvetica" font-size="12px" text-anchor="middle">
                    ...
                </text>
            </switch>
        </g>
        <rect x="2" y="0" width="1650" height="300" fill-opacity="0.4" fill="#36454f" stroke="none" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 603px; height: 1px; padding-top: 105px; margin-left: 19px;">
                        <div data-drawio-colors="color: #B85450; " style="box-sizing: border-box; font-size: 0px; text-align: left;">
                            <div style="display: inline-block; font-size: 16px; font-family: Helvetica; color: rgb(184, 84, 80); line-height: 1.2; pointer-events: none; font-weight: bold; white-space: normal; overflow-wrap: normal;">
                                Stage: Add attention module output and current tensor
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="19" y="110" fill="#B85450" font-family="Helvetica" font-size="16px" font-weight="bold">
                    Stage: Add attention module output and current tensor
                </text>
            </switch>
        </g>
        <rect x="644" y="545" width="270" height="56" rx="8.4" ry="8.4" fill="#008a00" stroke="#005700" pointer-events="none"/>
        <path d="M 648.35 548.36 M 827.35 548.36 M 827.35 567.36 M 648.35 567.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 555px; margin-left: 650px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 15px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                <b style="border-color: var(--border-color); font-size: 12px;">
                                    <span style="font-weight: 400; font-size: 12px;">
                                        Tensor:
                                    </span>
                                    attention output
                                </b>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="650" y="567" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    Tensor: attention output
                </text>
            </switch>
        </g>
        <path d="M 827.35 548.36 M 909.35 548.36 M 909.35 567.36 M 827.35 567.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 555px; margin-left: 827px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 15px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                BFloat16
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="907" y="567" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    BFloat16
                </text>
            </switch>
        </g>
        <path d="M 648.35 567.36 M 827.35 567.36 M 827.35 598.36 M 648.35 598.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 595px; margin-left: 650px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (sequenceLength, Dim)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="650" y="595" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    (sequenceLength, Dim)
                </text>
            </switch>
        </g>
        <path d="M 827.35 567.36 M 909.35 567.36 M 909.35 598.36 M 827.35 598.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 595px; margin-left: 827px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (22, 4096)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="907" y="595" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    (22, 4096)
                </text>
            </switch>
        </g>
        <path d="M 987 665 L 987 643 Q 987 633 977 633 L 789 633 Q 779 633 779 623 L 779 609.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 779 603.24 L 783 611.24 L 779 609.24 L 775 611.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 1px; height: 1px; padding-top: 635px; margin-left: 831px;">
                        <div data-drawio-colors="color: #6A6A6A; background-color: rgb(255, 255, 255); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 11px; font-family: Helvetica; color: rgb(106, 106, 106); line-height: 1.2; pointer-events: none; background-color: rgb(255, 255, 255); white-space: nowrap;">
                                <div style="border-color: var(--border-color);">
                                    <br/>
                                </div>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="831" y="638" fill="#6A6A6A" font-family="Helvetica" font-size="11px" text-anchor="middle">
                </text>
            </switch>
        </g>
        <rect x="852" y="843" width="270" height="56" rx="8.4" ry="8.4" fill="#008a00" stroke="#005700" pointer-events="none"/>
        <path d="M 856.35 846.36 M 1035.35 846.36 M 1035.35 865.36 M 856.35 865.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 853px; margin-left: 858px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 15px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                <b style="border-color: var(--border-color); font-size: 12px;">
                                    <span style="font-weight: 400; font-size: 12px;">
                                        Tensor:
                                    </span>
                                    x
                                </b>
                                (Input)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="858" y="865" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    Tensor: x (Input)
                </text>
            </switch>
        </g>
        <path d="M 1035.35 846.36 M 1117.35 846.36 M 1117.35 865.36 M 1035.35 865.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 853px; margin-left: 1035px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 15px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                BFloat16
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="1115" y="865" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    BFloat16
                </text>
            </switch>
        </g>
        <path d="M 856.35 865.36 M 1035.35 865.36 M 1035.35 896.36 M 856.35 896.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 893px; margin-left: 858px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (sequenceLength, Dim)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="858" y="893" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    (sequenceLength, Dim)
                </text>
            </switch>
        </g>
        <path d="M 1035.35 865.36 M 1117.35 865.36 M 1117.35 896.36 M 1035.35 896.36" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 893px; margin-left: 1035px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (22, 4096)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="1115" y="893" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    (22, 4096)
                </text>
            </switch>
        </g>
        <rect x="852" y="793" width="270" height="30" rx="4.5" ry="4.5" fill="#f8cecc" stroke="#b85450" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 268px; height: 1px; padding-top: 808px; margin-left: 853px;">
                        <div data-drawio-colors="color: #B85450; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(184, 84, 80); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                ...
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="987" y="812" fill="#B85450" font-family="Helvetica" font-size="12px" text-anchor="middle">
                    ...
                </text>
            </switch>
        </g>
        <path d="M 987 793 L 987 776.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 987 770.24 L 991 778.24 L 987 776.24 L 983 778.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 1px; height: 1px; padding-top: 777px; margin-left: 987px;">
                        <div data-drawio-colors="color: #6A6A6A; background-color: rgb(255, 255, 255); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 11px; font-family: Helvetica; color: rgb(106, 106, 106); line-height: 1.2; pointer-events: none; background-color: rgb(255, 255, 255); white-space: nowrap;">
                                <div style="border-color: var(--border-color);">
                                    <br/>
                                </div>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="987" y="780" fill="#6A6A6A" font-family="Helvetica" font-size="11px" text-anchor="middle">
                </text>
            </switch>
        </g>
        <path d="M 987 843 L 987 831.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 987 825.24 L 991 833.24 L 987 831.24 L 983 833.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 1px; height: 1px; padding-top: 830px; margin-left: 987px;">
                        <div data-drawio-colors="color: #6A6A6A; background-color: rgb(255, 255, 255); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 11px; font-family: Helvetica; color: rgb(106, 106, 106); line-height: 1.2; pointer-events: none; background-color: rgb(255, 255, 255); white-space: nowrap;">
                                <div style="border-color: var(--border-color);">
                                    <br/>
                                </div>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="987" y="833" fill="#6A6A6A" font-family="Helvetica" font-size="11px" text-anchor="middle">
                </text>
            </switch>
        </g>
        <path d="M 852 871 L 112 871 Q 102 871 102 861 L 102 416 Q 102 406 112 406 L 630.16 406" fill="none" stroke="#c73500" stroke-width="5" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 638.41 406 L 627.41 411.5 L 630.16 406 L 627.41 400.5 Z" fill="#c73500" stroke="#c73500" stroke-width="5" stroke-miterlimit="10" pointer-events="none"/>
        <rect x="644" y="391" width="270" height="30" rx="4.5" ry="4.5" fill="#1ba1e2" stroke="#006eaf" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 268px; height: 1px; padding-top: 406px; margin-left: 645px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                Add x and h tensors
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="779" y="410" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="middle">
                    Add x and h tensors
                </text>
            </switch>
        </g>
        <rect x="644" y="311.81" width="270" height="56" rx="8.4" ry="8.4" fill="#008a00" stroke="#005700" pointer-events="none"/>
        <path d="M 648.35 315.17 M 827.35 315.17 M 827.35 334.17 M 648.35 334.17" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 322px; margin-left: 650px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 15px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                <b style="border-color: var(--border-color); font-size: 12px;">
                                    <span style="font-weight: 400; font-size: 12px;">
                                        Tensor: h
                                    </span>
                                </b>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="650" y="334" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    Tensor: h
                </text>
            </switch>
        </g>
        <path d="M 827.35 315.17 M 909.35 315.17 M 909.35 334.17 M 827.35 334.17" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 322px; margin-left: 827px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 15px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                BFloat16
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="907" y="334" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    BFloat16
                </text>
            </switch>
        </g>
        <path d="M 648.35 334.17 M 827.35 334.17 M 827.35 365.17 M 648.35 365.17" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-start; width: 177px; height: 1px; padding-top: 362px; margin-left: 650px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: left; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (sequenceLength, Dim)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="650" y="362" fill="#F0F0F0" font-family="Helvetica" font-size="12px">
                    (sequenceLength, Dim)
                </text>
            </switch>
        </g>
        <path d="M 827.35 334.17 M 909.35 334.17 M 909.35 365.17 M 827.35 365.17" fill="none" stroke="#ffffff" stroke-linecap="square" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-end; justify-content: unsafe flex-end; width: 80px; height: 1px; padding-top: 362px; margin-left: 827px;">
                        <div data-drawio-colors="color: #F0F0F0; " style="box-sizing: border-box; font-size: 0px; text-align: right; max-height: 27px; overflow: hidden;">
                            <div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(240, 240, 240); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                (22, 4096)
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="907" y="362" fill="#F0F0F0" font-family="Helvetica" font-size="12px" text-anchor="end">
                    (22, 4096)
                </text>
            </switch>
        </g>
        <path d="M 779 439 L 779 429.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 779 423.24 L 783 431.24 L 779 429.24 L 775 431.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 779 391 L 779 376.05" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 779 370.05 L 783 378.05 L 779 376.05 L 775 378.05 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 987 933 L 987 907.24" fill="none" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <path d="M 987 901.24 L 991 909.24 L 987 907.24 L 983 909.24 Z" fill="#6a6a6a" stroke="#6a6a6a" stroke-width="2" stroke-miterlimit="10" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 1px; height: 1px; padding-top: 910px; margin-left: 987px;">
                        <div data-drawio-colors="color: #6A6A6A; background-color: rgb(255, 255, 255); " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 11px; font-family: Helvetica; color: rgb(106, 106, 106); line-height: 1.2; pointer-events: none; background-color: rgb(255, 255, 255); white-space: nowrap;">
                                <div style="border-color: var(--border-color);">
                                    <br/>
                                </div>
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="987" y="913" fill="#6A6A6A" font-family="Helvetica" font-size="11px" text-anchor="middle">
                </text>
            </switch>
        </g>
        <rect x="2" y="910" width="1650" height="60" fill-opacity="0.4" fill="#36454f" stroke="none" pointer-events="none"/>
        <rect x="-246.74" y="547.91" width="610.47" height="113.62" fill-opacity="0.4" fill="#f8cecc" stroke="#b85450" stroke-opacity="0.4" transform="rotate(270,58.49,604.72)" pointer-events="none"/>
        <g transform="translate(-0.5 -0.5)rotate(270 58.49499999999989 604.7199999999998)">
            <switch>
                <foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;">
                    <div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 608px; height: 1px; padding-top: 605px; margin-left: -246px;">
                        <div data-drawio-colors="color: #B85450; " style="box-sizing: border-box; font-size: 0px; text-align: center;">
                            <div style="display: inline-block; font-size: 24px; font-family: Helvetica; color: rgb(184, 84, 80); line-height: 1.2; pointer-events: none; white-space: normal; overflow-wrap: normal;">
                                You are here
                            </div>
                        </div>
                    </div>
                </foreignObject>
                <text x="58" y="612" fill="#B85450" font-family="Helvetica" font-size="24px" text-anchor="middle">
                    You are here
                </text>
            </switch>
        </g>
        <path d="M 72 599.47 L 104 599.47 L 104 587.22 L 112 604.72 L 104 622.22 L 104 609.97 L 72 609.97 Z" fill-opacity="0.4" fill="#b85450" stroke="rgb(0, 0, 0)" stroke-opacity="0.4" stroke-miterlimit="10" pointer-events="none"/>
    </g>
    <switch>
        <g requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"/>
        <a transform="translate(0,-5)" xlink:href="https://www.diagrams.net/doc/faq/svg-export-text-problems" target="_blank">
            <text text-anchor="middle" font-size="10px" x="50%" y="100%">
                Text is not SVG - cannot display
            </text>
        </a>
    </switch>
</svg>