diff --git a/scripts/adapters.dockerfile b/scripts/adapters.dockerfile index 7dac26b94992..3e9809bf4312 100644 --- a/scripts/adapters.dockerfile +++ b/scripts/adapters.dockerfile @@ -30,14 +30,16 @@ RUN mamba create -y --name adapters python=3.8 SHELL ["mamba", "run", "-n", "adapters", "/bin/bash", "-c"] RUN pip install https://github.com/googleapis/storage-testbench/archive/refs/tags/v0.36.0.tar.gz -RUN mamba install -y nodejs openjdk +RUN mamba install -y nodejs RUN npm install -g azurite ENV HADOOP_HOME=/usr/local/hadoop \ HADOOP_ROOT_LOGGER="WARN,DRFA" \ LC_ALL=C \ - LIBHDFS3_CONF=/velox/scripts/hdfs-client.xml \ - PATH=/usr/local/hadoop/bin:${PATH} + PATH=/usr/local/hadoop/bin:${PATH} \ + JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk \ + PATH=/usr/lib/jvm/java-1.8.0-openjdk/bin:${PATH} -ENTRYPOINT ["/bin/bash", "-c", "source /opt/rh/gcc-toolset-12/enable && exec \"$@\"", "--"] +COPY scripts/setup-classpath.sh / +ENTRYPOINT ["/bin/bash", "-c", "source /set_classpath.sh && source /opt/rh/gcc-toolset-12/enable && exec \"$@\"", "--"] CMD ["/bin/bash"] diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh index a8c832ea54c5..6928c477199c 100755 --- a/scripts/setup-adapters.sh +++ b/scripts/setup-adapters.sh @@ -141,8 +141,12 @@ function install_hdfs_deps { sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt sed -i "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMake/Platform.cmake # Dependencies for Hadoop testing - wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop + wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0.tar.gz hadoop cp -a hadoop /usr/local/ + wget -P /usr/local/hadoop/share/hadoop/common/lib/ https://repo1.maven.org/maven2/junit/junit/4.11/junit-4.11.jar + + yum install -y java-1.8.0-openjdk-devel + fi cmake_install $libhdfs3_dir } diff --git a/scripts/setup-classpath.sh b/scripts/setup-classpath.sh new file mode 100644 index 000000000000..e52184d92138 --- /dev/null +++ b/scripts/setup-classpath.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export CLASSPATH=`/usr/local/hadoop/bin/hdfs classpath --glob`