From f358edc9678abb01b6059aeac96e7d9a3393a8fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Apr 2024 12:27:33 +0100 Subject: [PATCH] Bump github/super-linter from 5 to 6 (#335) * Bump github/super-linter from 5 to 6 Bumps [github/super-linter](https://github.com/github/super-linter) from 5 to 6. - [Release notes](https://github.com/github/super-linter/releases) - [Changelog](https://github.com/github/super-linter/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/super-linter/compare/v5...v6) --- updated-dependencies: - dependency-name: github/super-linter dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Add explicit reference to checkout * Shell format * markdown fixes * Disable CHECKOV * Trying to get this sorted out * Disable linters * Use the right name of the variable * Site no longer available --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Enol Fernandez --- .github/CODE_OF_CONDUCT.md | 8 +-- .github/workflows/lint.yml | 10 +++- cloud-info/ams-wrapper.sh | 95 ++++++++++++++++--------------- cloud-info/publisher.sh | 10 ++-- deploy/deploy.sh | 64 ++++++++++----------- sites/GSI-LCG2.yaml | 25 -------- validate.sh | 114 +++++++++++++++++-------------------- 7 files changed, 151 insertions(+), 175 deletions(-) delete mode 100644 sites/GSI-LCG2.yaml diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md index 92a0f068..33a68541 100644 --- a/.github/CODE_OF_CONDUCT.md +++ b/.github/CODE_OF_CONDUCT.md @@ -68,7 +68,7 @@ offensive, or harmful. This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail +representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. @@ -76,9 +76,9 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the EGI Foundation team at contact@egi.eu. The team will -review and investigate all complaints, and will respond in a way that it deems -appropriate to the circumstances. The team is obligated to maintain +reported by contacting the [EGI Foundation team](mailto:contact@egi.eu). The +team will review and investigate all complaints, and will respond in a way that +it deems appropriate to the circumstances. The team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b5640659..d0ae11ab 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,7 +1,7 @@ --- name: Lint -on: [pull_request, push] +on: pull_request jobs: super-lint: @@ -12,9 +12,15 @@ jobs: # Checks out a copy of your repository on the ubuntu-latest machine - name: Checkout code uses: actions/checkout@v4 + with: + # Full git history needed to get proper list of changed files + fetch-depth: 0 # Runs the Super-Linter action - name: Run Super-Linter - uses: github/super-linter/slim@v5 + uses: github/super-linter/slim@v6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Disabling these two for the moment, should be enabled later + VALIDATE_CHECKOV: false + VALIDATE_PYTHON_PYLINT: false diff --git a/cloud-info/ams-wrapper.sh b/cloud-info/ams-wrapper.sh index b28a758d..7f1c0b77 100755 --- a/cloud-info/ams-wrapper.sh +++ b/cloud-info/ams-wrapper.sh @@ -10,9 +10,9 @@ GOCDB_ID=$(python -c "from __future__ import print_function; \ timeout=60)['gocdb_id'], end='')") if test "$AMS_TOKEN_FILE" != ""; then - AMS_TOKEN=$(cat "$AMS_TOKEN_FILE") -elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then - AMS_TOKEN=$(python -c "from argo_ams_library import ArgoMessagingService; \ + AMS_TOKEN=$(cat "$AMS_TOKEN_FILE") +elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then + AMS_TOKEN=$(python -c "from argo_ams_library import ArgoMessagingService; \ ams = ArgoMessagingService(endpoint='$AMS_HOST', \ project='$AMS_PROJECT', \ cert='$HOSTCERT', \ @@ -21,16 +21,18 @@ elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then fi if test "$SITE_NAME" = ""; then - SITE_NAME="$(yq -r .site.name "$CLOUD_INFO_CONFIG" | tr "." "-")" + SITE_NAME="$(yq -r .site.name "$CLOUD_INFO_CONFIG" | tr "." "-")" fi SITE_TOPIC=$(echo "$SITE_NAME" | tr "." "-") AMS_TOPIC="SITE_${SITE_TOPIC}_ENDPOINT_${GOCDB_ID}" # exit if TOPIC is not available. -curl -f "https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC?key=$AMS_TOKEN" > /dev/null 2>&1 \ - || (echo "Topic $AMS_TOPIC is not avaiable, aborting!"; false) - +curl -f "https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC?key=$AMS_TOKEN" >/dev/null 2>&1 || + ( + echo "Topic $AMS_TOPIC is not avaiable, aborting!" + false + ) # Attempt to generate the site configuration AUTO_CONFIG_PATH="$(mktemp -d)" @@ -41,57 +43,60 @@ export CHECKIN_SECRETS_FILE="$CHECKIN_SECRETS_PATH/secrets.yaml" export ACCESS_TOKEN_FILE="$AUTO_CONFIG_PATH/token.yaml" USE_ACCESS_TOKEN=0 if token-generator; then - # TODO(enolfc): even if this belows fails, we should use access token as it will provide - # access to more projects - if SECRETS_FILE="$ACCESS_TOKEN_FILE" config-generator > "$AUTO_CONFIG_PATH/site.yaml"; then - # this worked, let's update the env - export CHECKIN_SECRETS_PATH="$AUTO_CONFIG_PATH/vos" - export CLOUD_INFO_CONFIG="$AUTO_CONFIG_PATH/site.yaml" - USE_ACCESS_TOKEN=1 - fi + # TODO(enolfc): even if this belows fails, we should use access token as it will provide + # access to more projects + if SECRETS_FILE="$ACCESS_TOKEN_FILE" config-generator >"$AUTO_CONFIG_PATH/site.yaml"; then + # this worked, let's update the env + export CHECKIN_SECRETS_PATH="$AUTO_CONFIG_PATH/vos" + export CLOUD_INFO_CONFIG="$AUTO_CONFIG_PATH/site.yaml" + USE_ACCESS_TOKEN=1 + fi fi # Any OS related parameter should be available as env variables if test "$CHECKIN_SECRETS_PATH" = ""; then - # Case 1: manual config - cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \ - --middleware "$CLOUD_INFO_MIDDLEWARE" \ - --ignore-share-errors \ - --format glue21 > cloud-info.out + # Case 1: manual config + cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \ + --middleware "$CLOUD_INFO_MIDDLEWARE" \ + --ignore-share-errors \ + --format glue21 >cloud-info.out elif test "$USE_ACCESS_TOKEN" -eq 1; then - # Case 2: access token style - cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \ - --middleware "$CLOUD_INFO_MIDDLEWARE" \ - --ignore-share-errors \ - --auth-refresher accesstoken \ - --format glue21 > cloud-info.out + # Case 2: access token style + cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \ + --middleware "$CLOUD_INFO_MIDDLEWARE" \ + --ignore-share-errors \ + --auth-refresher accesstoken \ + --format glue21 >cloud-info.out else - # Let's use the service account directly on the info provider - CHECKIN_DISCOVERY="https://aai.egi.eu/auth/realms/egi/.well-known/openid-configuration" - CLIENT_ID="$(yq -r '.fedcloudops.client_id' < "$CHECKIN_SECRETS_FILE")" - CLIENT_SECRET="$(yq -r '.fedcloudops.client_secret' < "$CHECKIN_SECRETS_FILE")" - cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \ - --middleware "$CLOUD_INFO_MIDDLEWARE" \ - --ignore-share-errors \ - --os-auth-type v3oidcclientcredentials \ - --os-discovery-endpoint "$CHECKIN_DISCOVERY" \ - --os-client-id "$CLIENT_ID" \ - --os-client-secret "$CLIENT_SECRET" \ - --os-access-token-type access_token \ - --os-openid-scope "openid profile eduperson_entitlement email" \ - --format glue21 > cloud-info.out + # Let's use the service account directly on the info provider + CHECKIN_DISCOVERY="https://aai.egi.eu/auth/realms/egi/.well-known/openid-configuration" + CLIENT_ID="$(yq -r '.fedcloudops.client_id' <"$CHECKIN_SECRETS_FILE")" + CLIENT_SECRET="$(yq -r '.fedcloudops.client_secret' <"$CHECKIN_SECRETS_FILE")" + cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \ + --middleware "$CLOUD_INFO_MIDDLEWARE" \ + --ignore-share-errors \ + --os-auth-type v3oidcclientcredentials \ + --os-discovery-endpoint "$CHECKIN_DISCOVERY" \ + --os-client-id "$CLIENT_ID" \ + --os-client-secret "$CLIENT_SECRET" \ + --os-access-token-type access_token \ + --os-openid-scope "openid profile eduperson_entitlement email" \ + --format glue21 >cloud-info.out fi # Fail if there are no shares -grep -q GLUE2ShareID cloud-info.out \ - || (echo "No share information available, aborting!"; false) +grep -q GLUE2ShareID cloud-info.out || + ( + echo "No share information available, aborting!" + false + ) # Publishing on our own as message is too large for some providers ARGO_URL="https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC:publish?key=$AMS_TOKEN" -printf '{"messages":[{"attributes":{},"data":"' > ams-payload -grep -v "UNKNOWN" cloud-info.out | grep -v "^#" | grep -v ": $" | gzip | base64 -w 0 >> ams-payload -printf '"}]}' >> ams-payload +printf '{"messages":[{"attributes":{},"data":"' >ams-payload +grep -v "UNKNOWN" cloud-info.out | grep -v "^#" | grep -v ": $" | gzip | base64 -w 0 >>ams-payload +printf '"}]}' >>ams-payload curl -X POST "$ARGO_URL" -H "content-type: application/json" -d @ams-payload diff --git a/cloud-info/publisher.sh b/cloud-info/publisher.sh index fe89b88f..0ba9e979 100755 --- a/cloud-info/publisher.sh +++ b/cloud-info/publisher.sh @@ -10,9 +10,9 @@ export CLOUD_INFO_MIDDLEWARE=openstack ams-wrapper.sh if [ -n "$OCCI_ENDPOINT" ]; then - # OCCI - export GOCDB_URL="$OCCI_ENDPOINT" - export GOCDB_SERVICE_TYPE=eu.egi.cloud.vm-management.occi - export CLOUD_INFO_MIDDLEWARE=ooi - ams-wrapper.sh + # OCCI + export GOCDB_URL="$OCCI_ENDPOINT" + export GOCDB_SERVICE_TYPE=eu.egi.cloud.vm-management.occi + export CLOUD_INFO_MIDDLEWARE=ooi + ams-wrapper.sh fi diff --git a/deploy/deploy.sh b/deploy/deploy.sh index eaff5587..2e7ab19b 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -11,21 +11,21 @@ SLACK_WEBHOOK_URL="$4" ansible-galaxy install git+https://github.com/EGI-Federation/ansible-role-fedcloud-ops.git -echo "cloud_info_image: \"ghcr.io/egi-federation/fedcloud-cloud-info:sha-$SHORT_SHA\"" >> extra-vars.yaml +echo "cloud_info_image: \"ghcr.io/egi-federation/fedcloud-cloud-info:sha-$SHORT_SHA\"" >>extra-vars.yaml # Configure! -if ansible-playbook -i inventory.yaml \ - --extra-vars @secrets.yaml \ - --extra-vars @extra-vars.yaml \ - --extra-vars @vos.yaml \ - playbook.yaml >ansible.log 2>&1 ; then - status_summary="success" - color="#6DBF59" - header="Successful deployment :rocket:" +if ansible-playbook -i inventory.yaml \ + --extra-vars @secrets.yaml \ + --extra-vars @extra-vars.yaml \ + --extra-vars @vos.yaml \ + playbook.yaml >ansible.log 2>&1; then + status_summary="success" + color="#6DBF59" + header="Successful deployment :rocket:" else - status_summary="fail" - color="#EA4F47" - header="Failed deployment :boom:" + status_summary="fail" + color="#EA4F47" + header="Failed deployment :boom:" fi # This is a temporary way to get the auto discovery working while we transition for all sites @@ -36,32 +36,32 @@ GITHUB_COMMIT_URL="https://api.github.com/repos/EGI-Federation/fedcloud-catchall # Find out PR we need to update ISSUE_NUMBER=$(curl \ - -H "Accept: application/vnd.github.groot-preview+json" \ - "$GITHUB_COMMIT_URL" | jq .[0].number) + -H "Accept: application/vnd.github.groot-preview+json" \ + "$GITHUB_COMMIT_URL" | jq .[0].number) GITHUB_ISSUE_URL="https://api.github.com/repos/EGI-Federation/fedcloud-catchall-operations/issues/$ISSUE_NUMBER/comments" { - echo "### Ansible deployment: \`$status_summary\`" - echo '
Deployment log' - echo - echo '```' - cat ansible.log - echo '```' - echo - echo '
' -} > github_body.txt -echo "{}" | jq --arg b "$(cat github_body.txt)" '{body: $b}' > github_body.json + echo "### Ansible deployment: \`$status_summary\`" + echo '
Deployment log' + echo + echo '```' + cat ansible.log + echo '```' + echo + echo '
' +} >github_body.txt +echo "{}" | jq --arg b "$(cat github_body.txt)" '{body: $b}' >github_body.json # Let GitHub know comment_url=$(curl -X POST \ - -H "Authorization: token $OAUTH_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - "$GITHUB_ISSUE_URL" \ - --data @github_body.json | \ - jq -r .html_url) + -H "Authorization: token $OAUTH_TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "$GITHUB_ISSUE_URL" \ + --data @github_body.json | + jq -r .html_url) -cat > slack_body.json << EOF +cat >slack_body.json < "$VO_LIST" +curl --silent "http://cclavoisier01.in2p3.fr:8080/lavoisier/VoList?accept=json" | + jq -r ".data[].name" >"$VO_LIST" # Get fedcloudclient sites FEDCLOUD_CLI_SITES=$(mktemp) curl "https://raw.githubusercontent.com/tdviet/fedcloudclient/master/config/sites.yaml" \ - > "$FEDCLOUD_CLI_SITES" + >"$FEDCLOUD_CLI_SITES" # Temp file for nova endpoint NOVA_ENDPOINT=$(mktemp) -for f in sites/*.yaml -do - goc_site=$(grep "^gocdb:" "$f" | cut -f2 -d":" | tr -d "[:space:]") - endpoint=$(grep "^endpoint:" "$f" | cut -f2- -d":" | tr -d "[:space:]") - printf "Searching for endpoint %s in %s site (%s)\n" \ - "$endpoint" "$goc_site" "$f" - curl --silent "$goc_method&sitename=$goc_site&service_type=org.openstack.nova" \ - > "$NOVA_ENDPOINT" - if ! grep -q "$goc_site" "$NOVA_ENDPOINT" - then - printf "\033[0;31m[ERROR] Site %s not found in GOC\033[0m\n" "$goc_site" - exit_value=1 - continue - fi - if ! grep -q "$endpoint" "$NOVA_ENDPOINT" - then - printf "\033[0;31m[ERROR] URL %s for %s not found in GOC\033[0m\n" \ - "$endpoint" "$goc_site" - exit_value=1 - else - printf "\033[0;32m[OK]\033[0m\n" - fi - # check if all VOs configured do exist - # Try to use FQAN - # So the VO that comes from the file, it will be either: - # - just the name of the VO - # - //some more extra/ - # - /VO=/some more stuff/ - for vo in $(yq -r ".vos[].name" < "$f" | cut -f2 -d"/" | sed "s/^VO=//") - do - if ! grep -q "^$vo\$" "$VO_LIST" - then - printf "\033[0;31m[ERROR] VO %s not found in ops portal\033[0m\n" \ - "$vo" - exit_value=1 - fi - done +for f in sites/*.yaml; do + goc_site=$(grep "^gocdb:" "$f" | cut -f2 -d":" | tr -d "[:space:]") + endpoint=$(grep "^endpoint:" "$f" | cut -f2- -d":" | tr -d "[:space:]") + printf "Searching for endpoint %s in %s site (%s)\n" \ + "$endpoint" "$goc_site" "$f" + curl --silent "$goc_method&sitename=$goc_site&service_type=org.openstack.nova" \ + >"$NOVA_ENDPOINT" + if ! grep -q "$goc_site" "$NOVA_ENDPOINT"; then + printf "\033[0;31m[ERROR] Site %s not found in GOC\033[0m\n" "$goc_site" + exit_value=1 + continue + fi + if ! grep -q "$endpoint" "$NOVA_ENDPOINT"; then + printf "\033[0;31m[ERROR] URL %s for %s not found in GOC\033[0m\n" \ + "$endpoint" "$goc_site" + exit_value=1 + else + printf "\033[0;32m[OK]\033[0m\n" + fi + # check if all VOs configured do exist + # Try to use FQAN + # So the VO that comes from the file, it will be either: + # - just the name of the VO + # - //some more extra/ + # - /VO=/some more stuff/ + for vo in $(yq -r ".vos[].name" <"$f" | cut -f2 -d"/" | sed "s/^VO=//"); do + if ! grep -q "^$vo\$" "$VO_LIST"; then + printf "\033[0;31m[ERROR] VO %s not found in ops portal\033[0m\n" \ + "$vo" + exit_value=1 + fi + done - # check if site is also on: - # https://github.com/tdviet/fedcloudclient/blob/master/config/sites.yaml - if ! grep -q "$f" "$FEDCLOUD_CLI_SITES" - then - printf "\033[0;31m[ERROR] Site %s not found in fedcloudclient\033[0m\n" "$goc_site" - exit_value=1 - fi + # check if site is also on: + # https://github.com/tdviet/fedcloudclient/blob/master/config/sites.yaml + if ! grep -q "$f" "$FEDCLOUD_CLI_SITES"; then + printf "\033[0;31m[ERROR] Site %s not found in fedcloudclient\033[0m\n" "$goc_site" + exit_value=1 + fi done -for site in $(yq -r '.[]' < "$FEDCLOUD_CLI_SITES") -do - if ! test -s "sites/$(basename "$site")" - then - printf "\033[0;31m[ERROR] Site %s not found in fedcloud-catchall-operations\033[0m\n" "$(basename "$site")" - exit_value=1 - fi +for site in $(yq -r '.[]' <"$FEDCLOUD_CLI_SITES"); do + if ! test -s "sites/$(basename "$site")"; then + printf "\033[0;31m[ERROR] Site %s not found in fedcloud-catchall-operations\033[0m\n" "$(basename "$site")" + exit_value=1 + fi done # check that the VO mappings are up to date according to ops portal -for vo in $(yq -r '.vos | keys[]' < vo-mappings.yaml | cut -f2 -d"/" | sed "s/^VO=//") -do - if ! grep -q "^$vo\$" "$VO_LIST" - then - printf "\033[0;31m[ERROR] VO %s not found in ops portal\033[0m\n" \ - "$vo" - exit_value=1 - fi +for vo in $(yq -r '.vos | keys[]'