Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Script Update to include latest Exclude files for JDK versions #4897

Closed
wants to merge 1 commit into from
Closed

Script Update to include latest Exclude files for JDK versions #4897

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
247 changes: 144 additions & 103 deletions jck/jck-semiauto-updater/jckupdater.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,27 @@ JCK_VERSION=""
JCK_UPDATE_NUMBER=""
JCK_GIT_REPO=""
ARTIFACTORY_DOWNLOAD_URL=""
ARTIFACTORY_API_URL=""
ARTIFACTORY_EXC_URL=""
ARTIFACTORY_TOKEN=""
JCK_REPO_NAME=""
GIT_USER=""
JCK_GIT_BRANCH="autoBranch"
GIT_TOKEN=""
JCK_FOLDER_SUFFIX=""
GIT_EXCLUDE_API_URL="https://api.github.ibm.com/repos/runtimes"
JAVA_SDK_PATH="$WORKSPACE/../../../../jdkbinary/j2sdk-image"

usage ()
{
echo 'Usage : jckupdater.sh [--jck_version|-j ] : Indicate JCK version to update.'
echo ' [--artifactory_token|-at] : Token to access JCK artifactory: https://eu.artifactory.swg-devops.com/artifactory/jim-jck-generic-local.'
echo ' [--artifactory_url|-au] : Artifactory server URL to download JCK material'
echo ' [--artifactory_api_url|-ap] : Artifactory API URL to fetch details on JCK material'
echo ' [--jck-repo|-repo] : JCK GIT repo to update.'
echo ' [--git_token|-gt] : Git API Token to create PR.'
echo ' [--jck_git_branch|-gb ] : Optional. JCK_GIT_BRANCH name to clone repo. Default is autoBranch'
echo ' [--java_home|-java] : Optional. JAVA_HOME path. '
echo ' [--java_sdk|-java] : Optional. JAVA_SDK_PATH path. '

}

Expand All @@ -44,8 +49,8 @@ parseCommandLineArgs()
"--jck-repo" | "-repo")
JCK_GIT_REPO="$1"; shift;;

"--java_home" | "-java" )
JAVA_HOME="$1"; shift;;
"--java_sdk" | "-java" )
JAVA_SDK_PATH="$1"; shift;;

"--git_token" | "-gt" )
GIT_TOKEN="$1"; shift;;
Expand All @@ -56,6 +61,12 @@ parseCommandLineArgs()
"--artifactory_url" | "-au")
ARTIFACTORY_DOWNLOAD_URL="$1"; shift;;

"--artifactory_api_url" | "-ap")
ARTIFACTORY_API_URL="$1"; shift;;

"--git_exclude_api_url" | "-eu")
GIT_EXCLUDE_API_URL="$1"; shift;;

"--help" | "-h" )
usage; exit 0;;

Expand All @@ -68,7 +79,7 @@ parseCommandLineArgs()
setup(){

echo ""
echo "JAVA_HOME=$JAVA_HOME"
echo "JAVA_SDK_PATH=$JAVA_SDK_PATH"
echo "WORKSPACE=$WORKSPACE"
echo "GIT_REPO=$GIT_REPO"
echo "JCK_VERSION=$JCK_VERSION"
Expand All @@ -95,12 +106,11 @@ setup(){
JCK_FOLDER_SUFFIX="$JCK_VERSION"a
fi
echo "JCK_FOLDER_SUFFIX=" $JCK_FOLDER_SUFFIX
#check if given ARTIFACTORY_DOWNLOAD_URL is complete till JCK/tck to download material
if ! echo "$ARTIFACTORY_DOWNLOAD_URL" | grep -q "${JCK}/tck/"; then
ARTIFACTORY_DOWNLOAD_URL="${ARTIFACTORY_DOWNLOAD_URL}/${JCK}/tck/"
fi

echo "ARTIFACTORY_DOWNLOAD_URL=$ARTIFACTORY_DOWNLOAD_URL"
ARTIFACTORY_JCK_DOWNLOAD_URL="${ARTIFACTORY_DOWNLOAD_URL}/${JCK}/"
ARTIFACTORY_EXC_URL="${ARTIFACTORY_API_URL}/${JCK}/exc/"
echo "ARTIFACTORY_JCK_DOWNLOAD_URL=$ARTIFACTORY_JCK_DOWNLOAD_URL"
echo "ARTIFACTORY_EXC_URL=$ARTIFACTORY_EXC_URL"
echo "GIT_EXCLUDE_API_URL=$GIT_EXCLUDE_API_URL"
echo ""

mkdir $WORKSPACE
Expand All @@ -109,39 +119,9 @@ setup(){
mkdir $WORKSPACE/jckmaterial
}


executeCmdWithRetry()
{
set +e
count=0
rt_code=-1
# when the command is not found (code 127), do not retry
while [ "$rt_code" != 0 ] && [ "$rt_code" != 127 ] && [ "$count" -le 5 ]
do
if [ "$count" -gt 0 ]; then
sleep_time=3
echo "error code: $rt_code. Sleep $sleep_time secs, then retry $count..."
sleep $sleep_time

echo "check for $1. If found, the file will be removed."
if [ "$1" != "" ] && [ -f "$1" ]; then
echo "remove $1 before retry..."
rm $1
fi
fi
echo "$2"
eval "$2"
rt_code=$?
count=$(( $count + 1 ))
done
set -e
return "$rt_code"
}



list() {
file_list=$(curl -ks -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} "${ARTIFACTORY_DOWNLOAD_URL}")
url="$1"
file_list=$(curl -ks -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} "${url}")

# Use grep to filter out the content within <a href=""> tags
file_names=$(echo "$file_list" | grep -o '<a href="[^"]*">' | sed 's/<a href="//;s/">//')
Expand All @@ -150,11 +130,11 @@ list() {
isLatestUpdate() {
cd $WORKSPACE/jckmaterial

list # get the JCK update number from artifactory
list "${ARTIFACTORY_JCK_DOWNLOAD_URL}tck/" # get the JCK update number from artifactory
last_file=""
for file in $file_names; do
last_file="$file"
done
for file in $file_names; do
last_file="$file"
done

JCK_UPDATE_NUMBER=$last_file

Expand All @@ -167,83 +147,141 @@ isLatestUpdate() {
curl -o "build.txt" $GIT_URL -H "Authorization: token $GIT_TOKEN"
echo -e "JCK version in build.txt:\n$(cat build.txt)\n\n"

#check if latest exclude file is available
getExcludeFiles
isExcludeUpdateNeeded=$?

#Check if material update is needed.
if grep -q "$JCK_WITHOUT_BACKSLASH" build.txt; then
isTestMaterialUpdateNeeded=1
else
isTestMaterialUpdateNeeded=0
fi

if [[ $isTestMaterialUpdateNeeded -eq 1 && $isExcludeUpdateNeeded -eq 1 ]]; then
echo " JCK$JCK_VERSION material is $JCK_WITHOUT_BACKSLASH in the repo $GIT_URL. It is up to date. No need to pull changes"
cleanup
exit 2
else
echo " JCK$JCK_VERSION $JCK_WITHOUT_BACKSLASH is latest and not in the repo $GIT_URL... Please proceed with download"
get_JAVA_SDK
else
if [ $isTestMaterialUpdateNeeded -eq 0 ]; then
echo " JCK$JCK_VERSION $JCK_WITHOUT_BACKSLASH is latest and not in the repo $GIT_URL... Please proceed with download"
else
echo " Latest exclude files are available for JCK$JCK_VERSION"
fi
getJCKSources
fi
}

getLastModifiedForFile() {
file="$1"
curl -s -H "X-JFrog-Art-Api:$ARTIFACTORY_TOKEN" "$ARTIFACTORY_EXC_URL$file" | awk -F'"' '/lastModified/ {print $4}'
}

getExcludeFiles() {
github_last_modified=$(curl -s -H "Authorization: token $GIT_TOKEN" "${GIT_EXCLUDE_API_URL}/JCK$JCK_VERSION-unzipped/commits?path=excludes/jck$JCK_FOLDER_SUFFIX.jtx" | awk -F'"date":' '{if(NF>1){gsub(/[",]/,"",$2); print $2; exit}}')
#check if file is present in github.
if [[ -n "${github_last_modified}" ]]; then
github_date_only=$(date -u -d "${github_last_modified}" "+%Y-%m-%d")
fi
artifactory_file_list=$(curl -s -H "X-JFrog-Art-Api:$ARTIFACTORY_TOKEN" "$ARTIFACTORY_EXC_URL" -s | \
awk -F'"uri" : "' '/"uri" : "\// { gsub(/".*$/, "", $2); print $2 }' | \
awk '{gsub(/^\//, ""); print}' | \
grep -v '/$' )

#if no exclude file is present
if [[ -z "$artifactory_file_list" ]]; then
echo "No exclude files present in Artifactory to update"
return 1
else
declare -a last_modified_dates
declare -a file_last_modified

echo "List of exclude files in Artifactory:"
echo "$artifactory_file_list"

#store all files one by one
files=()
while IFS= read -r line; do
files+=("$line")
done <<< "$artifactory_file_list"

for file in "${files[@]}"; do
artifactory_date=$(date -d "$(getLastModifiedForFile "$file")" "+%Y-%-m-%d")
last_modified_dates+=("$artifactory_date")
file_last_modified["$artifactory_date"]=$file
done

artifactory_latest_modified=$(printf "%s\n" "${last_modified_dates[@]}" | sort -rV | head -1)
latest_file_name="${file_last_modified[$artifactory_latest_modified]}"
formatted_date=$(date -d "$artifactory_latest_modified" "+%Y-%m-%d") # reconvert month to value with leading zero for comparison
echo "Latest exclude file: $latest_file_name"
echo "Last Modified Date in GitHub: ${github_date_only}"
echo "Last Modified Date in Artifactory: ${formatted_date}"

# Compare and find the latest last modified date only if file is present in github. Else update exclude file
if [[ -n "${github_last_modified}" && ("${github_date_only}" > "${formatted_date}" || "${github_date_only}" == "${formatted_date}" ) ]]; then
echo "No need to update exclude file in GitHub"
return 1
else
echo "Update exclude file with $latest_file_name from Artifactory"
return 0
fi
fi

}

## Download directly from given URL under current folder
getJCKSources() {
cd $WORKSPACE/jckmaterial
echo "remove build.txt file after comparison"
rm -rf build.txt
echo "download jck materials..."

ARTIFACTORY_DOWNLOAD_URL=$ARTIFACTORY_DOWNLOAD_URL$JCK_UPDATE_NUMBER
echo $ARTIFACTORY_DOWNLOAD_URL
#download latest exclude file
if [ $isExcludeUpdateNeeded -eq 0 ]; then
curl -OLJSk -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} ${ARTIFACTORY_JCK_DOWNLOAD_URL}exc/$latest_file_name && unzip -o "$latest_file_name"
fi

list #get list of files to download

IFS=$'\n' read -r -d '' -a file_names_array <<< "$file_names"
if [ $isTestMaterialUpdateNeeded -eq 0 ]; then
ARTIFACTORY_JCK_UPDATE_NUMBER_URL=${ARTIFACTORY_JCK_DOWNLOAD_URL}tck/$JCK_UPDATE_NUMBER
echo $ARTIFACTORY_JCK_UPDATE_NUMBER_URL

if [ "${ARTIFACTORY_DOWNLOAD_URL}" != "" ]; then
for file in "${file_names_array[@]:1}"; do
url="$ARTIFACTORY_DOWNLOAD_URL$file"
executeCmdWithRetry "${file##*/}" "_ENCODE_FILE_NEW=UNTAGGED curl -OLJSk -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} $url"
list "$ARTIFACTORY_JCK_UPDATE_NUMBER_URL" #get list of files to download

rt_code=$?
if [ $rt_code != 0 ]; then
echo "curl error code: $rt_code"
echo "Failed to retrieve $file. This is what we received of the file and MD5 sum:"
ls -ld $file
IFS=$'\n' read -r -d '' -a file_names_array <<< "$file_names"

exit 1
fi
done
fi
}
if [ "${ARTIFACTORY_JCK_UPDATE_NUMBER_URL}" != "" ]; then
for file in "${file_names_array[@]:1}"; do
url="$ARTIFACTORY_JCK_UPDATE_NUMBER_URL$file"
_ENCODE_FILE_NEW=UNTAGGED curl -OLJSk -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} $url
rt_code=$?
if [ $rt_code != 0 ]; then
echo "curl error code: $rt_code"
echo "Failed to retrieve $file. This is what we received of the file and MD5 sum:"
ls -ld $file

#install Java
get_JAVA_SDK(){
if [[ $JAVA_HOME = "" ]] ; then
cd $WORKSPACE/../../../../jdkbinary/j2sdk-image
JAVA_SDK_PATH="$(pwd)"
echo $JAVA_SDK_PATH
$JAVA_SDK_PATH/bin/java -version
exit 1
fi
done
fi
fi
}

#Unpack downloaded jar files
extract() {
cd $WORKSPACE/jckmaterial

echo "install downloaded resources"

echo "install downloaded resources"
for f in $WORKSPACE/jckmaterial/*.jar; do
echo "Unpacking $f:"

#using default java on machine for local
if [[ $JAVA_HOME != "" ]] ; then
$JAVA_HOME/bin/java -jar $f -install shell_scripts -o $WORKSPACE/unpackjck
else
$JAVA_SDK_PATH/bin/java -jar $f -install shell_scripts -o $WORKSPACE/unpackjck
fi

$JAVA_SDK_PATH/bin/java -jar $f -install shell_scripts -o $WORKSPACE/unpackjck
done
cd $WORKSPACE/unpackjck
ls -la
echo "completed unpack of jck resources"
}

#Clone GIT branch.
gitClone()
{
gitClone() {
echo "Checking out [email protected]:$GIT_USER/$JCK_REPO_NAME and reset to [email protected]:runtimes/$JCK_REPO_NAME main branch in dir $GIT_REPO"
cd $GIT_REPO
git init
Expand All @@ -258,27 +296,28 @@ gitClone()

#Move unpacked files to GIT repository
copyFilestoGITRepo() {

cd $GIT_REPO
rm -rf JCK-compiler-$JCK_FOLDER_SUFFIX
rm -rf JCK-runtime-$JCK_FOLDER_SUFFIX
rm -rf JCK-devtools-$JCK_FOLDER_SUFFIX
rm -rf headers
rm -rf natives
echo "copy unpacked JCK files from $WORKSPACE/unpackjck to local GIT dir $GIT_REPO"

cd $WORKSPACE/unpackjck
for file in $WORKSPACE/unpackjck/*; do
echo $file
cp -rf $file $GIT_REPO
done
if [ $isTestMaterialUpdateNeeded -eq 0 ]; then
rm -rf JCK-compiler-$JCK_FOLDER_SUFFIX
rm -rf JCK-runtime-$JCK_FOLDER_SUFFIX
rm -rf JCK-devtools-$JCK_FOLDER_SUFFIX
rm -rf headers
rm -rf natives
echo "copy unpacked JCK files from $WORKSPACE/unpackjck to local GIT dir $GIT_REPO"

cd $WORKSPACE/unpackjck
for file in $WORKSPACE/unpackjck/*; do
echo $file
cp -rf $file $GIT_REPO
done
fi

#copy remaining file like .jtx, .kfl .html to GIT Repo
for file in $WORKSPACE/jckmaterial/*; do
if [[ "$file" != *.zip ]] && [[ "$file" != *.gz* ]] && [[ "$file" != *.jar ]];then
if [[ "$file" == *.kfl* ]] || [[ "$file" == *.jtx* ]]; then
echo "Copy $file to $GIT_REPO"
cp -rf $file $GIT_REPO/excludes
mkdir -p $GIT_REPO/excludes && cp -rf $file $GIT_REPO/excludes
else
cp -rf $file $GIT_REPO
fi
Expand Down Expand Up @@ -354,7 +393,9 @@ if [ "$JCK_VERSION" != "" ] && [ "$JCK_GIT_REPO" != "" ] && [ "$GIT_TOKEN" != ""
cleanup
setup
isLatestUpdate
extract
if [ $isTestMaterialUpdateNeeded -eq 0 ]; then
extract
fi
gitClone
copyFilestoGITRepo
checkChangesAndCommit
Expand Down
Loading
Loading