diff --git a/.github/actions/setup-build-cuda/action.yml b/.github/actions/setup-build-cuda/action.yml
index 824be1bd6b..d3e06893ab 100644
--- a/.github/actions/setup-build-cuda/action.yml
+++ b/.github/actions/setup-build-cuda/action.yml
@@ -26,12 +26,14 @@ runs:
         TORCH_CUDA_DEFAULT = "121"  # pytorch 2.4.1
         # https://github.com/Jimver/cuda-toolkit/blob/master/src/links/linux-links.ts
         full_version, install_script = {
+          "126": ("12.6.3", "https://developer.download.nvidia.com/compute/cuda/12.6.3/local_installers/cuda_12.6.3_560.35.05_linux.run"),
           "124": ("12.4.1", "https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run"),
           "121": ("12.1.0", "https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda_12.1.0_530.30.02_linux.run"),
           "118": ("11.8.0", "https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run"),
           "6.0": ("6.0.2", "https://repo.radeon.com/amdgpu-install/6.0.2/rhel/8.9/amdgpu-install-6.0.60002-1.el8.noarch.rpm"),
           "6.1": ("6.1.2", "https://repo.radeon.com/amdgpu-install/6.1.3/rhel/8.9/amdgpu-install-6.1.60103-1.el8.noarch.rpm"),
-          "6.2": ("6.2.3", "https://repo.radeon.com/amdgpu-install/6.2.3/rhel/8.9/amdgpu-install-6.2.60203-1.el8.noarch.rpm"),
+          "6.2": ("6.2.4", "https://repo.radeon.com/amdgpu-install/6.2.4/rhel/8.9/amdgpu-install-6.2.60204-1.el8.noarch.rpm"),
+          "6.3": ("6.3.1", "https://repo.radeon.com/amdgpu-install/6.3.1/rhel/8.9/amdgpu-install-6.3.60301-1.el8.noarch.rpm"),
         }[cushort]
         with open(os.environ['GITHUB_OUTPUT'], "r+") as fp:
           fp.write("CUDA_VERSION=" + full_version + "\n")
diff --git a/.github/workflows/rocm_build.yml b/.github/workflows/rocm_build.yml
index 37fe17b4ec..ee21e21a90 100644
--- a/.github/workflows/rocm_build.yml
+++ b/.github/workflows/rocm_build.yml
@@ -24,7 +24,7 @@ jobs:
         python: ['3.11']
         torch_version: ['2.5.1']
         toolkit_type: ['rocm']
-        toolkit_short_version: ['6.1', '6.2']
+        toolkit_short_version: ['6.1', '6.2', '6.3']
 
     uses: ./.github/workflows/wheels_build.yml
     if: github.repository == 'rocm/xformers'
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index db6ba6572a..04c7197496 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -31,8 +31,8 @@ jobs:
         PY_VERSIONS = ['3.9', '3.10', '3.11', '3.12']
         # NOTE: Don't forget to update `upload_pt`'s matrix
         # when changing the CUDA/ROCM versions below!
-        CU_VERSIONS = ['118', '121', '124']
-        ROCM_VERSIONS = ["6.1"] # <- 6.0 broken in `manylinux_2_28`
+        CU_VERSIONS = ['118', '121', '124', '126']
+        ROCM_VERSIONS = ['6.1', '6.2', '6.3'] # <- 6.0 broken in `manylinux_2_28`
         PY_CU = list(itertools.product(PY_VERSIONS, CU_VERSIONS))
         PY_ROCM = list(itertools.product(PY_VERSIONS, ROCM_VERSIONS))
         print("Full matrix PY_CU", PY_CU)
@@ -42,7 +42,7 @@ jobs:
           for cu in CU_VERSIONS[1:]:
             PY_CU.append((PY_VERSIONS[-1], cu))
           print("Limited matrix PY_CU", PY_CU)
-          PY_ROCM = [(PY_VERSIONS[-1], ROCM_VERSIONS[-1])]
+          PY_ROCM = [(PY_VERSIONS[-1], ROCM_VERSIONS[-2])] # Always last version is upcoming version
 
         include = []
         for os in ['8-core-ubuntu', 'windows-8-core']:
@@ -111,6 +111,7 @@ jobs:
           - cu121
           - cu124
           - rocm6.1
+          - rocm6.2
     uses: ./.github/workflows/wheels_upload_s3.yml
     with:
       aws_role: "arn:aws:iam::749337293305:role/pytorch_bot_uploader_role"
@@ -118,4 +119,3 @@ jobs:
       aws_s3_cp_extra_args: --acl public-read
       filter: "*torch2.5.1+${{ matrix.suffix }}*"
       execute: ${{ github.repository == 'facebookresearch/xformers' && github.ref_type == 'tag' }}
-