Node group workers on demand #4242
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy_localhost_AllInOne | |
| on: | |
| push: | |
| branches: [ '**' ] | |
| tags-ignore: [ '**' ] | |
| jobs: | |
| versioning: | |
| name: "Versioning" | |
| runs-on: ubuntu-latest | |
| if: "!contains(github.event.head_commit.message, '[skip ci]')" | |
| outputs: | |
| version: ${{ steps.genver.outputs.version }} | |
| steps: | |
| - name: Get type of VM | |
| run: | | |
| lscpu | |
| uname -a | |
| grep MemTotal /proc/meminfo | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| fetch-depth: 0 | |
| - name: Generate Version | |
| id: genver | |
| run: | | |
| VERSION=$(docker run --rm -v $(pwd):/repo codacy/git-version /bin/git-version --folder=/repo --release-branch=release --dev-branch=main) | |
| echo "version=$VERSION" >> $GITHUB_OUTPUT | |
| echo $VERSION | |
| echo "Version :" >> $GITHUB_STEP_SUMMARY | |
| echo "$VERSION" >> $GITHUB_STEP_SUMMARY | |
| AdminGUI: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Get type of VM | |
| run: | | |
| lscpu | |
| uname -a | |
| grep MemTotal /proc/meminfo | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| fetch-depth: 0 | |
| - name: Install Dependencies | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/dependencies@main | |
| with: | |
| docker: true | |
| terraform: true | |
| k3s: true | |
| aws: true | |
| - name: Deploy | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/deploy@main | |
| with: | |
| type: localhost | |
| log-suffix: admin-gui | |
| - name: Test ping AdminGUI | |
| shell: bash | |
| run: | | |
| set -ex | |
| export ADMIN_API_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.admin_api_url') | |
| export ADMIN_APP_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.admin_app_url') | |
| : "Add -f again when a success endpoint has been implemented" | |
| curl -sSL "${ADMIN_API_URL}" -o /dev/null || true "in v2.7.2, there is no api endpoint" | |
| curl -fsSL "${ADMIN_APP_URL}" -o /dev/null || true "in v2.7.2, there is no app endpoint" | |
| - name: Show logs | |
| if: always() | |
| run: | | |
| set -e | |
| mkdir -p logs/infra/generated \ | |
| logs/infra/tfstates \ | |
| logs/app | |
| if [ -e infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json ] ; then | |
| cp infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json logs/infra/generated | |
| fi | |
| make -C infrastructure/quick-deploy/localhost/all-in-one/ state-pull > logs/infra/tfstates/armonik-terraform.tfstate | |
| sudo find /var/log/pods/ -maxdepth 1 -iname 'armonik_*' -exec cp -rL '{}' logs/app/ ';' | |
| sudo chown $USER -R logs | |
| tar -czf admin-gui.tar.gz logs | |
| aws s3 cp admin-gui.tar.gz s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/armonik-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/admin-gui.tar.gz | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_EC2_METADATA_DISABLED: true | |
| - name: Destroy deployment | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/destroy@main | |
| with: | |
| type: localhost | |
| coreStream: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - name: Get type of VM | |
| run: | | |
| lscpu | |
| uname -a | |
| grep MemTotal /proc/meminfo | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| fetch-depth: 0 | |
| - name: Get Core Tag | |
| id: core | |
| run: | | |
| TAG=$(cat versions.tfvars.json | jq -r '.armonik_versions.core') | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| - name: Install Dependencies | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/dependencies@main | |
| with: | |
| docker: true | |
| terraform: true | |
| k3s: true | |
| aws: true | |
| - name: Deploy | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/deploy@main | |
| with: | |
| type: localhost | |
| log-suffix: core-stream | |
| - name: Test Core Stream | |
| shell: bash | |
| run: | | |
| set -ex | |
| export CONTROL_PLANE_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.control_plane_url') | |
| docker run --rm \ | |
| -e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \ | |
| -e Partition="stream" \ | |
| dockerhubaneo/armonik_core_stream_test_client:${{ steps.core.outputs.tag }} | |
| - name: Show logs | |
| if: always() | |
| run: | | |
| set -e | |
| mkdir -p core-stream/infra/generated \ | |
| core-stream/infra/tfstates \ | |
| core-stream/app | |
| if [ -e infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json ] ; then | |
| cp infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json core-stream/infra/generated | |
| fi | |
| make -C infrastructure/quick-deploy/localhost/all-in-one/ state-pull > core-stream/infra/tfstates/armonik-terraform.tfstate | |
| sudo find /var/log/pods/ -maxdepth 1 -iname 'armonik_*' -exec cp -rL '{}' core-stream/app/ ';' | |
| sudo chown $USER -R core-stream | |
| tar -czf core-stream.tar.gz core-stream | |
| aws s3 cp core-stream.tar.gz s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/armonik-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/core-stream.tar.gz | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_EC2_METADATA_DISABLED: true | |
| - name: Destroy deployment | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/destroy@main | |
| with: | |
| type: localhost | |
| htcMock: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - name: Get type of VM | |
| run: | | |
| lscpu | |
| uname -a | |
| grep MemTotal /proc/meminfo | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| fetch-depth: 0 | |
| - name: Get Core Tag | |
| id: core | |
| run: | | |
| TAG=$(cat versions.tfvars.json | jq -r '.armonik_versions.core') | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| - name: Install Dependencies | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/dependencies@main | |
| with: | |
| docker: true | |
| terraform: true | |
| k3s: true | |
| aws: true | |
| - name: Deploy | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/deploy@main | |
| with: | |
| type: localhost | |
| log-suffix: htc-mock | |
| - name: Test HTC Mock | |
| shell: bash | |
| run: | | |
| set -ex | |
| export CONTROL_PLANE_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.control_plane_url') | |
| docker run --rm \ | |
| -e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \ | |
| -e HtcMock__NTasks=2000 \ | |
| -e HtcMock__TotalCalculationTime=00:00:50.0 \ | |
| -e HtcMock__DataSize=50 \ | |
| -e HtcMock__MemorySize=50 \ | |
| -e HtcMock__SubTasksLevels=5 \ | |
| -e HtcMock__EnableUseLowMem=false \ | |
| -e HtcMock__EnableSmallOutput=false \ | |
| -e HtcMock__EnableFastCompute=false \ | |
| -e HtcMock__Partition="htcmock" \ | |
| dockerhubaneo/armonik_core_htcmock_test_client:${{ steps.core.outputs.tag }} | |
| - name: Test HTC Mock - 1 level | |
| shell: bash | |
| run: | | |
| set -ex | |
| export CONTROL_PLANE_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.control_plane_url') | |
| docker run --rm \ | |
| -e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \ | |
| -e HtcMock__NTasks=1000 \ | |
| -e HtcMock__TotalCalculationTime=00:00:00.100 \ | |
| -e HtcMock__DataSize=1 \ | |
| -e HtcMock__MemorySize=1 \ | |
| -e HtcMock__SubTasksLevels=1 \ | |
| -e HtcMock__EnableUseLowMem=true \ | |
| -e HtcMock__EnableSmallOutput=true \ | |
| -e HtcMock__EnableFastCompute=true \ | |
| -e HtcMock__Partition="htcmock" \ | |
| dockerhubaneo/armonik_core_htcmock_test_client:${{ steps.core.outputs.tag }} | |
| - name: Test HTC Mock - 5 levels | |
| shell: bash | |
| run: | | |
| set -ex | |
| export CONTROL_PLANE_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.control_plane_url') | |
| docker run --rm \ | |
| -e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \ | |
| -e HtcMock__NTasks=1000 \ | |
| -e HtcMock__TotalCalculationTime=00:00:00.100 \ | |
| -e HtcMock__DataSize=1 \ | |
| -e HtcMock__MemorySize=1 \ | |
| -e HtcMock__SubTasksLevels=5 \ | |
| -e HtcMock__EnableUseLowMem=true \ | |
| -e HtcMock__EnableSmallOutput=true \ | |
| -e HtcMock__EnableFastCompute=true \ | |
| -e HtcMock__Partition="htcmock" \ | |
| dockerhubaneo/armonik_core_htcmock_test_client:${{ steps.core.outputs.tag }} | |
| - name: Test HTC Mock - 10 levels | |
| shell: bash | |
| run: | | |
| set -ex | |
| export CONTROL_PLANE_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.control_plane_url') | |
| docker run --rm \ | |
| -e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \ | |
| -e HtcMock__NTasks=1000 \ | |
| -e HtcMock__TotalCalculationTime=00:00:00.100 \ | |
| -e HtcMock__DataSize=1 \ | |
| -e HtcMock__MemorySize=1 \ | |
| -e HtcMock__SubTasksLevels=10 \ | |
| -e HtcMock__EnableUseLowMem=true \ | |
| -e HtcMock__EnableSmallOutput=true \ | |
| -e HtcMock__EnableFastCompute=true \ | |
| -e HtcMock__Partition="htcmock" \ | |
| dockerhubaneo/armonik_core_htcmock_test_client:${{ steps.core.outputs.tag }} | |
| - name: Show logs | |
| if: always() | |
| run: | | |
| set -e | |
| mkdir -p htcmock/infra/generated \ | |
| htcmock/infra/tfstates \ | |
| htcmock/app | |
| if [ -e infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json ] ; then | |
| cp infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json htcmock/infra/generated | |
| fi | |
| make -C infrastructure/quick-deploy/localhost/all-in-one/ state-pull > htcmock/infra/tfstates/armonik-terraform.tfstate | |
| sudo find /var/log/pods/ -maxdepth 1 -iname 'armonik_*' -exec cp -rL '{}' htcmock/app/ ';' | |
| sudo chown $USER -R htcmock | |
| tar -czf htcmock.tar.gz htcmock | |
| aws s3 cp htcmock.tar.gz s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/armonik-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock.tar.gz | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_EC2_METADATA_DISABLED: true | |
| - name: Destroy deployment | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/destroy@main | |
| with: | |
| type: localhost | |
| bench: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Get type of VM | |
| run: | | |
| lscpu | |
| uname -a | |
| grep MemTotal /proc/meminfo | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| fetch-depth: 0 | |
| - name: Get Core Tag | |
| id: core | |
| run: | | |
| TAG=$(cat versions.tfvars.json | jq -r '.armonik_versions.core') | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| - name: Install Dependencies | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/dependencies@main | |
| with: | |
| docker: true | |
| terraform: true | |
| k3s: true | |
| aws: true | |
| - name: Deploy | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/deploy@main | |
| with: | |
| type: localhost | |
| log-suffix: bench | |
| - name: Test Bench - many tasks (200) | |
| shell: bash | |
| run: | | |
| set -ex | |
| export CONTROL_PLANE_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.control_plane_url') | |
| docker run --rm \ | |
| -e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \ | |
| -e BenchOptions__NTasks=200 \ | |
| -e BenchOptions__TaskDurationMs=100 \ | |
| -e BenchOptions__Partition="bench"\ | |
| dockerhubaneo/armonik_core_bench_test_client:${{ steps.core.outputs.tag }} | |
| - name: Test Bench - long tasks (10s) | |
| shell: bash | |
| run: | | |
| set -ex | |
| export CONTROL_PLANE_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.control_plane_url') | |
| docker run --rm \ | |
| -e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \ | |
| -e BenchOptions__NTasks=2 \ | |
| -e BenchOptions__TaskDurationMs=10000 \ | |
| -e BenchOptions__Partition="bench"\ | |
| dockerhubaneo/armonik_core_bench_test_client:${{ steps.core.outputs.tag }} | |
| - name: Test Bench - large payloads (10MB) | |
| shell: bash | |
| run: | | |
| set -ex | |
| export CONTROL_PLANE_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.control_plane_url') | |
| docker run --rm \ | |
| -e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \ | |
| -e BenchOptions__NTasks=10 \ | |
| -e BenchOptions__TaskDurationMs=10 \ | |
| -e BenchOptions__ResultSize=1 \ | |
| -e BenchOptions__PayloadSize=10000 \ | |
| -e BenchOptions__Partition="bench"\ | |
| dockerhubaneo/armonik_core_bench_test_client:${{ steps.core.outputs.tag }} | |
| - name: Test Bench - large results (10MB) | |
| shell: bash | |
| run: | | |
| set -ex | |
| export CONTROL_PLANE_URL=$(cat infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json | jq -r '.armonik.control_plane_url') | |
| docker run --rm \ | |
| -e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \ | |
| -e BenchOptions__NTasks=10 \ | |
| -e BenchOptions__TaskDurationMs=10 \ | |
| -e BenchOptions__ResultSize=10000 \ | |
| -e BenchOptions__PayloadSize=1 \ | |
| -e BenchOptions__Partition="bench"\ | |
| dockerhubaneo/armonik_core_bench_test_client:${{ steps.core.outputs.tag }} | |
| - name: Show logs | |
| if: always() | |
| run: | | |
| set -e | |
| mkdir -p bench/infra/generated \ | |
| bench/infra/tfstates \ | |
| bench/app | |
| if [ -e infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json ] ; then | |
| cp infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json bench/infra/generated | |
| fi | |
| make -C infrastructure/quick-deploy/localhost/all-in-one/ state-pull > bench/infra/tfstates/armonik-terraform.tfstate | |
| sudo find /var/log/pods/ -maxdepth 1 -iname 'armonik_*' -exec cp -rL '{}' bench/app/ ';' | |
| sudo chown $USER -R bench | |
| tar -czf bench.tar.gz bench | |
| aws s3 cp bench.tar.gz s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/armonik-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/bench.tar.gz | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_EC2_METADATA_DISABLED: true | |
| - name: Destroy deployment | |
| uses: aneoconsulting/ArmoniK.Action.Deploy/destroy@main | |
| with: | |
| type: localhost | |