diff --git a/dev/release/post-08-docs.sh b/dev/release/post-08-docs.sh
index c59f9b96857a6..58a462551f199 100755
--- a/dev/release/post-08-docs.sh
+++ b/dev/release/post-08-docs.sh
@@ -72,13 +72,28 @@ fi
 # delete current stable docs and restore all previous versioned docs
 rm -rf docs/*
 git checkout "${versioned_paths[@]}"
+# Download and untar released docs in a temp folder
+rm -rf docs_new
+mkdir docs_new
+pushd docs_new
 curl \
   --fail \
   --location \
   --remote-name \
   https://apache.jfrog.io/artifactory/arrow/docs/${version}/docs.tar.gz
 tar xvf docs.tar.gz
-rm -f docs.tar.gz
+# Update DOCUMENTATION_OPTIONS.show_version_warning_banner
+find docs \
+  -type f \
+  -exec \
+    sed -i.bak \
+      -e "s/DOCUMENTATION_OPTIONS.show_version_warning_banner = true/DOCUMENTATION_OPTIONS.show_version_warning_banner = false/g" \
+      {} \;
+find ./ -name '*.bak' -delete
+popd
+mv docs_new/docs/* docs/
+rm -rf docs_new
+
 if [ "$is_major_release" = "yes" ] ; then
   previous_series=${previous_version%.*}
   mv docs_temp docs/${previous_series}