diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 2c400b073988..2a528feffc11 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -112,7 +112,7 @@ jobs:
       - name: Install dependencies
         shell: Rscript {0}
         run: |
-          source("./R-package/tests/helper_scripts/install_deps.R")
+          source("./ops/script/install_deps.R")
       - name: Run lintr
         run: |
           MAKEFLAGS="-j$(nproc)" R CMD INSTALL R-package/
diff --git a/.github/workflows/r_nold.yml b/.github/workflows/r_nold.yml
index 89f079fc1df0..6bfd1b3275a6 100644
--- a/.github/workflows/r_nold.yml
+++ b/.github/workflows/r_nold.yml
@@ -26,16 +26,13 @@ jobs:
         shell: bash
         run: |
           apt update && apt install libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev libglpk-dev libxml2-dev libharfbuzz-dev libfribidi-dev git -y
-
       - uses: actions/checkout@v4
         with:
           submodules: 'true'
-
       - name: Install dependencies
         shell: bash -l {0}
         run: |
-          /tmp/R-devel/bin/Rscript -e "source('./R-package/tests/helper_scripts/install_deps.R')"
-
+          /tmp/R-devel/bin/Rscript -e "source('./ops/script/install_deps.R')"
       - name: Run R tests
         shell: bash
         run: |
diff --git a/.github/workflows/r_tests.yml b/.github/workflows/r_tests.yml
index 312040f1d11f..f06798da88ff 100644
--- a/.github/workflows/r_tests.yml
+++ b/.github/workflows/r_tests.yml
@@ -97,7 +97,7 @@ jobs:
       - name: Install dependencies
         shell: Rscript {0}
         run: |
-          source("./R-package/tests/helper_scripts/install_deps.R")
+          source("./ops/script/install_deps.R")
       - name: Test R
         shell: powershell
         run: |
@@ -125,7 +125,7 @@ jobs:
       - name: Install dependencies
         shell: bash -l {0}
         run: |
-          Rscript -e "source('./R-package/tests/helper_scripts/install_deps.R')"
+          Rscript -e "source('./ops/script/install_deps.R')"
       - name: Test R
         shell: bash -l {0}
         run: |
diff --git a/R-package/configure b/R-package/configure
index 59daf32cb27d..5dfe0925e592 100755
--- a/R-package/configure
+++ b/R-package/configure
@@ -647,6 +647,7 @@ ac_header_cxx_list=
 ac_subst_vars='LTLIBOBJS
 LIBOBJS
 DMLC_DEFS
+FILESYSTEM_LIB
 BACKTRACE_LIB
 ENDIAN_FLAG
 OPENMP_LIB
@@ -2935,6 +2936,38 @@ else $as_nop
 fi
 
 
+### In older versions of gcc implementation of std::filesystem is kept
+### in separate stdc++fs library. Link it explicitly
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if std::filesystem requires linking stdc++fs" >&5
+printf %s "checking if std::filesystem requires linking stdc++fs... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+        #include <filesystem>
+        int main() {
+            std::filesystem::create_directory("/dev/null");
+        }
+
+_ACEOF
+if ac_fn_cxx_try_link "$LINENO"
+then :
+  ac_cv_fs_stdlib=no
+else $as_nop
+  ac_cv_fs_stdlib=yes
+
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+if test "x$ac_cv_fs_stdlib" = xyes; then
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+    FILESYSTEM_LIB='-lstdc++fs'
+else
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+    FILESYSTEM_LIB=''
+fi
+
 ### Check whether fopen64 is available
 ac_fn_cxx_check_func "$LINENO" "fopen64" "ac_cv_func_fopen64"
 if test "x$ac_cv_func_fopen64" = xyes
@@ -3281,6 +3314,7 @@ fi
 
 
 
+
 ac_config_files="$ac_config_files src/Makevars"
 
 ac_config_headers="$ac_config_headers config.h"
diff --git a/R-package/configure.ac b/R-package/configure.ac
index 31219e0bf31c..b209dd9b3f38 100644
--- a/R-package/configure.ac
+++ b/R-package/configure.ac
@@ -31,6 +31,27 @@ AC_CHECK_LIB([execinfo], [backtrace],
              [BACKTRACE_LIB=-lexecinfo],
              [BACKTRACE_LIB=''; DMLC_DEFS="$DMLC_DEFS -DDMLC_LOG_STACK_TRACE=0"])
 
+### In older versions of gcc implementation of std::filesystem is kept
+### in separate stdc++fs library. Link it explicitly
+AC_MSG_CHECKING([if std::filesystem requires linking stdc++fs])
+AC_LINK_IFELSE(
+    [AC_LANG_SOURCE([
+        #include <filesystem>
+        int main() {
+            std::filesystem::create_directory("/dev/null");
+        }
+    ])],
+    [ac_cv_fs_stdlib=no],
+    [ac_cv_fs_stdlib=yes]
+)
+if test "x$ac_cv_fs_stdlib" = xyes; then
+    AC_MSG_RESULT(yes)
+    FILESYSTEM_LIB='-lstdc++fs'
+else
+    AC_MSG_RESULT(no)
+    FILESYSTEM_LIB=''
+fi
+
 ### Check whether fopen64 is available
 AC_CHECK_FUNC(fopen64, [], [DMLC_DEFS="$DMLC_DEFS -DDMLC_USE_FOPEN64=0"])
 
@@ -90,6 +111,7 @@ AC_SUBST(OPENMP_CXXFLAGS)
 AC_SUBST(OPENMP_LIB)
 AC_SUBST(ENDIAN_FLAG)
 AC_SUBST(BACKTRACE_LIB)
+AC_SUBST(FILESYSTEM_LIB)
 AC_SUBST(DMLC_DEFS)
 AC_CONFIG_FILES([src/Makevars])
 AC_CONFIG_HEADERS([config.h])
diff --git a/R-package/src/Makevars.in b/R-package/src/Makevars.in
index 846ab8652fc3..b901a0620d9e 100644
--- a/R-package/src/Makevars.in
+++ b/R-package/src/Makevars.in
@@ -36,6 +36,7 @@ PKG_LIBS = \
     @OPENMP_LIB@ \
     @ENDIAN_FLAG@ \
     @BACKTRACE_LIB@ \
+    @FILESYSTEM_LIB@ \
     -pthread
 
 OBJECTS= \
diff --git a/ops/docker/ci_container.yml b/ops/docker/ci_container.yml
index b41246aa5365..df48885633ff 100644
--- a/ops/docker/ci_container.yml
+++ b/ops/docker/ci_container.yml
@@ -25,7 +25,6 @@ xgb-ci.gpu_build_r_rockylinux8:
   container_def: gpu_build_r_rockylinux8
   build_args:
     CUDA_VERSION_ARG: "12.4.1"
-    R_VERSION_ARG: "4.4.2"
 
 xgb-ci.alpine:
   container_def: alpine
diff --git a/ops/docker/dockerfile/Dockerfile.gpu_build_r_rockylinux8 b/ops/docker/dockerfile/Dockerfile.gpu_build_r_rockylinux8
index e7d816b62f80..60ba91e093d6 100644
--- a/ops/docker/dockerfile/Dockerfile.gpu_build_r_rockylinux8
+++ b/ops/docker/dockerfile/Dockerfile.gpu_build_r_rockylinux8
@@ -1,24 +1,29 @@
 ARG CUDA_VERSION_ARG=notset
 FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
 ARG CUDA_VERSION_ARG
-ARG R_VERSION_ARG
 
 # Install all basic requirements
 RUN \
     curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
         > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
     dnf -y update && \
-    dnf -y install dnf-plugins-core && \
+    dnf -y install dnf-plugins-core epel-release && \
     dnf config-manager --set-enabled powertools && \
-    dnf install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
-                   xorg-x11-server-devel openssl-devel zlib-devel bzip2-devel xz-devel libxml2-devel \
-                   pcre2-devel libcurl-devel texlive-* pandoc \
+    dnf install -y tar unzip wget xz git which ninja-build R texlive-* libxml2-devel \
                    gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
                    gcc-toolset-10-gcc-gfortran gcc-toolset-10-libquadmath-devel \
-                   gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel
+                   gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel \
+                   gcc-toolset-10-annobin qpdf && \
+    # Workaround to get the R XML package to build
+    ln -s /opt/rh/gcc-toolset-10/root/usr/lib/gcc/x86_64-redhat-linux/10/plugin/annobin.so \
+          /opt/rh/gcc-toolset-10/root/usr/lib/gcc/x86_64-redhat-linux/10/plugin/gcc-annobin.so
 
-ENV PATH=/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION_ARG/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/$R_VERSION_ARG/lib64:$LD_LIBRARY_PATH
+# Install R packages
+COPY script/install_deps.R /scripts/
+RUN Rscript /scripts/install_deps.R
+
+ENV PATH=/usr/local/ninja:/opt/software/packages/bin:$PATH
+ENV LD_LIBRARY_PATH=/opt/software/packages/lib:$LD_LIBRARY_PATH
 ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
 ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
 ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
@@ -26,12 +31,17 @@ ENV F77=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
 ENV FC=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
 
 RUN \
-    wget -nv -nc https://cran.r-project.org/src/base/R-4/R-$R_VERSION_ARG.tar.gz  && \
-    tar xf R-$R_VERSION_ARG.tar.gz  && \
-    cd R-$R_VERSION_ARG  && \
-    ./configure --prefix=/opt/R/$R_VERSION_ARG --enable-R-shlib --with-pcrel  && \
-    make -j$(nproc)  && \
-    make install && \
+    # checkbashisms
+    wget -nv -O devscripts.tar.xz https://ftp.debian.org/debian/pool/main/d/devscripts/devscripts_2.24.4.tar.xz && \
+    tar xvf ./devscripts.tar.xz -O "devscripts/scripts/checkbashisms.pl" > ./checkbashisms.pl && \
+    chmod +x ./checkbashisms.pl && \
+    mv ./checkbashisms.pl /usr/local/bin/checkbashisms && \
+    # Pandoc
+    wget -nv -O pandoc.tar.gz https://github.com/jgm/pandoc/releases/download/3.5/pandoc-3.5-linux-amd64.tar.gz && \
+    tar xvzf pandoc.tar.gz --strip-components 1 -C /usr/local/ && \
+    # Python
+    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
+    bash conda.sh -b -p /opt/miniforge && \
     # CMake
     wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
     bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr
diff --git a/ops/docker/extract_build_args.jq b/ops/docker/extract_build_args.jq
index b35240edb626..491fb9ce3ed2 100644
--- a/ops/docker/extract_build_args.jq
+++ b/ops/docker/extract_build_args.jq
@@ -1,7 +1,8 @@
 ## Example input:
-##   xgb-ci.gpu_build_r_rockylinux8
+##   xgb-ci.gpu_build_rockylinux8
 ## Example output:
-##   --build-arg CUDA_VERSION_ARG=12.4.1 --build-arg R_VERSION_ARG=4.3.2
+##   --build-arg CUDA_VERSION_ARG=12.4.1 --build-arg NCCL_VERSION_ARG=2.23.4-1
+##   --build-arg RAPIDS_VERSION_ARG=24.10
 def compute_build_args($input; $container_id):
   $input |
   .[$container_id] |
diff --git a/ops/pipeline/test-rpkg-impl.sh b/ops/pipeline/test-rpkg-impl.sh
index 82fe64cf9bef..11e81c228f79 100755
--- a/ops/pipeline/test-rpkg-impl.sh
+++ b/ops/pipeline/test-rpkg-impl.sh
@@ -10,9 +10,10 @@ then
 fi
 
 build_tool="$1"
+shift 1
 
-set -x
+source /opt/miniforge/bin/activate
 
+set -x
 R -e 'dir.create(Sys.getenv("R_LIBS_USER"), recursive = TRUE)'
-Rscript ./R-package/tests/helper_scripts/install_deps.R
 python3 ops/script/test_r_package.py --build-tool="${build_tool}"
diff --git a/R-package/tests/helper_scripts/install_deps.R b/ops/script/install_deps.R
similarity index 100%
rename from R-package/tests/helper_scripts/install_deps.R
rename to ops/script/install_deps.R