Release integrations #1005
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release integrations | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| jobs: | |
| prepare-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.prepare-matrix.outputs.INTEGRATIONS_MATRIX }} | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.DOCKER_MACHINE_USER }} | |
| password: ${{ secrets.DOCKER_MACHINE_TOKEN }} | |
| - name: Prepare matrix | |
| id: prepare-matrix | |
| run: | | |
| integrations_to_release=() | |
| # Get the list of integrations | |
| files=$(find integrations/*/.port -name "spec.yaml") | |
| for file in $files; do | |
| folder=$(dirname "$file") | |
| type=$(grep -E '^name = ".*"' "$folder/../pyproject.toml" | cut -d'"' -f2) | |
| # Get the version from pyproject.toml | |
| version=$(grep -E '^version = ".*"' "$folder/../pyproject.toml" | cut -d'"' -f2) | |
| # Check if the version exists in the ghcr.io registry | |
| rc=0 | |
| docker manifest inspect ghcr.io/port-labs/port-ocean-$type:$version > /dev/null 2>&1 || rc=$? | |
| if [ $rc -eq 0 ]; then | |
| echo "Image already exists in $repository: port-ocean-$type:$version" | |
| else | |
| integrations_to_release+=($file) | |
| fi | |
| done | |
| echo $(echo ${integrations_to_release[@]} | jq -R -c 'split(" ")') | |
| echo "INTEGRATIONS_MATRIX=$(echo ${integrations_to_release[@]} | jq -R -c 'split(" ")')" >> $GITHUB_OUTPUT | |
| release-integration: | |
| runs-on: ubuntu-latest | |
| if: needs.prepare-matrix.outputs.matrix != '[]' | |
| outputs: | |
| is_dev_version: ${{ steps.prepare_tags.outputs.is_dev_version }} | |
| permissions: | |
| packages: write | |
| contents: read | |
| needs: [prepare-matrix] | |
| strategy: | |
| # limit the number of parallel jobs to avoid hitting the ghcr.io rate limit | |
| max-parallel: 5 | |
| matrix: | |
| integration: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: linux/amd64,linux/arm64 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.DOCKER_MACHINE_USER }} | |
| password: ${{ secrets.DOCKER_MACHINE_TOKEN }} | |
| - name: Prepare Docker images tags | |
| id: prepare_tags | |
| run: | | |
| current_integration_spec=${{ matrix.integration }} | |
| folder=$(dirname "$current_integration_spec") | |
| context_dir=$(dirname "$folder") | |
| echo "context_dir=$context_dir" >> $GITHUB_OUTPUT | |
| version=$(grep -E '^version = ".*"' "$folder/../pyproject.toml" | cut -d'"' -f2) | |
| type=$(grep -E '^name = ".*"' "$folder/../pyproject.toml" | cut -d'"' -f2) | |
| echo "version=$version" >> $GITHUB_OUTPUT | |
| dockerfile_path=integrations/_infra/Dockerfile | |
| if test -e $folder/../Dockerfile; then | |
| dockerfile_path=$folder/../Dockerfile | |
| fi | |
| echo "dockerfile_path=$dockerfile_path" >> $GITHUB_OUTPUT | |
| # Check if the 'version' variable contains any character other than digits and "." | |
| if [[ ! "$version" =~ ^[0-9.]+$ ]]; then | |
| # If 'version' contains non-numeric and non-dot characters, skip building 'latest' tag | |
| tags="ghcr.io/port-labs/port-ocean-$type:$version" | |
| echo "tags=$tags" >> $GITHUB_OUTPUT | |
| echo "is_dev_version=true" >> $GITHUB_OUTPUT | |
| echo "Version contains non-numeric characters. Building without 'latest' tag." | |
| else | |
| # If 'version' contains only digits and dots, build with both 'latest' and version tags | |
| tags="ghcr.io/port-labs/port-ocean-$type:$version,ghcr.io/port-labs/port-ocean-$type:latest" | |
| echo "tags=$tags" >> $GITHUB_OUTPUT | |
| echo "is_dev_version=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Build and push | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ${{ steps.prepare_tags.outputs.dockerfile_path }} | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| tags: ${{ steps.prepare_tags.outputs.tags }} | |
| build-args: | | |
| BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }} | |
| INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }} | |
| upload-specs: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| contents: read | |
| needs: release-integration | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials 🔒 | |
| id: aws-credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Upload specifications to s3 | |
| run: | | |
| # Temporary file | |
| temp_file="temp.yaml" | |
| # Output file name | |
| index_file="index.json" | |
| # AWS S3 bucket details | |
| aws_s3_bucket="ocean-registry" | |
| # Fetch existing index file or create empty one of not exists | |
| if aws s3 ls "s3://$aws_s3_bucket/$index_file" >/dev/null 2>&1; then | |
| aws s3 cp "s3://$aws_s3_bucket/$index_file" $index_file | |
| echo "Successfully fetched global index file from s3 bucket." | |
| else | |
| echo "Index file does not exist in the S3 bucket, Creating new one..." | |
| echo "[]" >"$index_file" | |
| fi | |
| # Find all ocean-spec.yaml files under the specified directory | |
| find integrations/*/.port -type f -name "spec.yaml" >file_list.txt | |
| while IFS= read -r file; do | |
| integration_dir=$(dirname "$file") | |
| integration_name=$(echo "$integration_dir" | awk -F'/' '{print $2}') | |
| # Extract the type for pyproject.toml | |
| type=$(grep -E '^name = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2) | |
| # Extract the version from pyproject.toml | |
| version=$(grep -E '^version = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2) | |
| integration_spec="$integration_name-$version.json" | |
| integration_dest="$integration_name/$integration_spec" | |
| integration_legacy_dest="$integration_name.json" # TODO: legacy support, remove once not in use | |
| # Convert YAML to JSON | |
| yq -o json "$file" >"$temp_file" | |
| # Add version attribute | |
| jq --arg type "$type" --arg version "$version" '. + {type: $type, version: $version}' "$temp_file" >"$integration_spec" | |
| if [[ ! ${version} =~ ^.+-dev$ ]]; then | |
| # Upload integration's version manifest to s3 | |
| aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_dest" | |
| aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_legacy_dest" # TODO: legacy support, remove once not in use | |
| echo "Successfully uploaded $integration_spec to s3 bucket." | |
| # Get the latest version of the current integration | |
| latest_version=$(jq --arg type "$type" -r '.[] | select(.type == $type) | .version' "$index_file") | |
| # Add integration's spec to global index file | |
| regexp="^[0-9.]+$" | |
| if [[ ! "$latest_version" ]]; then | |
| # Add new integration spec if latest tag doesn't exist | |
| jq --argjson new_spec "[$(cat "$integration_spec")]" '. += $new_spec' "$index_file" >"$temp_file" | |
| mv "$temp_file" "$index_file" | |
| elif [[ ! "$latest_version" =~ $regexp ]] || [[ "$version" =~ $regexp ]]; then | |
| # Override global index file if released non-dev version or integration doesn't have non-dev latest tag | |
| jq --argjson updated_spec "$(cat "$integration_spec")" --arg type "$type" \ | |
| 'map(if .type == $type then $updated_spec else . end)' "$index_file" >"$temp_file" | |
| mv "$temp_file" "$index_file" | |
| fi | |
| # Upload global index file to s3 | |
| aws s3 cp "$index_file" "s3://$aws_s3_bucket/$index_file" | |
| echo "Successfully uploaded $index_file to s3 bucket." | |
| else | |
| echo "Did not upload integration spec since the version was in development ('${version}')" | |
| fi | |
| done <file_list.txt |