Skip to content

Commit

Permalink
Merge branch 'main' into jg/staticinit
Browse files Browse the repository at this point in the history
  • Loading branch information
aneojgurhem authored Dec 19, 2024
2 parents a8e1249 + 220164c commit 8b83d78
Show file tree
Hide file tree
Showing 169 changed files with 3,648 additions and 2,018 deletions.
51 changes: 51 additions & 0 deletions .docs/content/0.installation/5.version-migration.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# How to migrate ArmoniK.Core dependencies during upgrade ?

## 0.28.x -> 0.29.x

### Database

This version changes the structure of a Result in the database. It introduces a new field called `OpaqueId` which holds the identifier of its associated value in the Object Storage. Previously, the ResultId was used. The following MongoDB request converts the ResultId into the OpaqueId to support the new implementation.

```js
db.Result.updateMany({},
[{
$addFields: {
OpaqueId: {
$function: {
body: function(data) {
const base64Chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
const bytes = [];
for (let i = 0; i < data.length; i++) {
bytes.push(data.charCodeAt(i));
}

let base64 = '';
let i = 0;
while (i < bytes.length) {
let byte1 = bytes[i++] || 0;
let byte2 = bytes[i++] || 0;
let byte3 = bytes[i++] || 0;

let enc1 = byte1 >> 2;
let enc2 = ((byte1 & 3) << 4) | (byte2 >> 4);
let enc3 = ((byte2 & 15) << 2) | (byte3 >> 6);
let enc4 = byte3 & 63;

if (isNaN(byte2)) {
enc3 = enc4 = 64;
} else if (isNaN(byte3)) {
enc4 = 64;
}

base64 += base64Chars[enc1] + base64Chars[enc2] + base64Chars[enc3] + base64Chars[enc4];
}

return BinData(0, base64);
},
args: ["$_id"],
lang: "js"
}
}
}
}])
```
2 changes: 1 addition & 1 deletion .docs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
},
"dependencies": {
"@aneoconsultingfr/armonik-docs-theme": "^0.6.12",
"mermaid": "^10.2.3",
"mermaid": "^11.4.0",
"nuxt": "^3.6.1"
},
"devDependencies": {
Expand Down
1,404 changes: 996 additions & 408 deletions .docs/pnpm-lock.yaml

Large diffs are not rendered by default.

97 changes: 67 additions & 30 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
version: ${{ steps.genver.outputs.version }}
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
fetch-depth: 0
Expand All @@ -40,20 +40,26 @@ jobs:
- Adaptors/MongoDB/tests
- Adaptors/Memory/tests
- Adaptors/S3/tests
- Adaptors/Embed/tests
os:
- ubuntu-latest
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true

- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup Terraform
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
with:
terraform_version: latest

- name: Setup just
run: |
Expand All @@ -67,12 +73,12 @@ jobs:
- name: Dotnet Restore
run: |
MONITOR_PREFIX="monitor/restore/" MONITOR_CD=${{ matrix.projects }} tools/retry.sh -w 60 -- tools/monitor.sh \
MONITOR_PREFIX="monitor/restore/" tools/retry.sh -w 60 -- tools/monitor.sh \
dotnet restore
- name: Dotnet Build
run: |
MONITOR_PREFIX="monitor/build/" MONITOR_CD=${{ matrix.projects }} tools/monitor.sh \
MONITOR_PREFIX="monitor/build/" tools/monitor.sh \
dotnet build
- name: Run tests
Expand Down Expand Up @@ -113,15 +119,20 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true

- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup Terraform
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
with:
terraform_version: latest

- name: Setup just
run: |
sudo snap install --edge --classic just
Expand All @@ -133,12 +144,12 @@ jobs:
- name: Dotnet Restore
run: |
MONITOR_PREFIX="monitor/restore/" MONITOR_CD=${{ matrix.projects }} tools/retry.sh -w 60 -- tools/monitor.sh \
MONITOR_PREFIX="monitor/restore/" tools/retry.sh -w 60 -- tools/monitor.sh \
dotnet restore
- name: Dotnet Build
run: |
MONITOR_PREFIX="monitor/build/" MONITOR_CD=${{ matrix.projects }} tools/monitor.sh \
MONITOR_PREFIX="monitor/build/" tools/monitor.sh \
dotnet build
- name: Run tests
Expand Down Expand Up @@ -169,24 +180,21 @@ jobs:
- Common/tests
- Adaptors/MongoDB/tests
- Adaptors/Memory/tests
- Adaptors/Embed/tests
fail-fast: false
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true

- name: Dotnet Restore
run: |
cd ${{ matrix.projects }}
dotnet restore
run: dotnet restore

- name: Dotnet Build
run: |
cd ${{ matrix.projects }}
dotnet build
run: dotnet build

- name: Run tests
run: |
Expand All @@ -205,7 +213,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true
Expand All @@ -222,7 +230,7 @@ jobs:
VERSION: ${{ needs.versionning.outputs.version }}
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}

Expand Down Expand Up @@ -258,7 +266,7 @@ jobs:
- worker=htcmock buildWorker
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}

Expand Down Expand Up @@ -306,7 +314,7 @@ jobs:
username: ${{ secrets.DOCKER_HUB_LOGIN }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: Analyze for critical and high CVEs
uses: docker/scout-action@cc6bf8dd03587425ef920278b3e2726ba8d791e8 # v1
uses: docker/scout-action@b23590dc1e4d09febc00cfcbc51e9e8c0f7ee9f3 # v1
with:
command: cves
image: "${{ matrix.image }}:${{ needs.versionning.outputs.version }}"
Expand Down Expand Up @@ -336,7 +344,7 @@ jobs:
- Verbose
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true
Expand All @@ -351,6 +359,8 @@ jobs:
- name: Setup Terraform
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
with:
terraform_version: latest

- name: Deploy Core
run: |
Expand Down Expand Up @@ -440,12 +450,18 @@ jobs:
- { queue: pubsub, object: minio, log-level: Information, cinit: true }
- { queue: sqs, object: minio, log-level: Information, cinit: true }

- { queue: activemq, object: embed, log-level: Information, cinit: true }
- { queue: rabbitmq, object: embed, log-level: Information, cinit: true }
- { queue: rabbitmq091, object: embed, log-level: Information, cinit: true }
- { queue: pubsub, object: embed, log-level: Information, cinit: true }
- { queue: sqs, object: embed, log-level: Information, cinit: true }

- { queue: activemq, object: redis, log-level: Information, cinit: false }

name: HtcMock ${{ matrix.target.queue }} ${{ matrix.target.object }} ${{ matrix.target.log-level }} ${{ matrix.target.cinit }}
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true
Expand All @@ -457,9 +473,11 @@ jobs:
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Setup Terraform
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
with:
terraform_version: latest

- name: Deploy Core
run: |
Expand Down Expand Up @@ -604,13 +622,17 @@ jobs:
name: Docker Windows
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true

- name: Setup Terraform
run: choco install terraform mongodb-shell 7zip just
- name: Setup Terraform and other tools
run: |
choco install terraform --version=1.10.1 -y
choco install mongodb-shell -y
choco install 7zip -y
choco install just -y
- name: Setup AWS cli
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4
Expand All @@ -623,6 +645,11 @@ jobs:
run: just -v tag=$env:VERSION object=local worker=htcmock ingress=false prometheus=false grafana=false seq=false queue=rabbitmq091 deploy
shell: powershell

- name: Pull image
run: docker pull dockerhubaneo/armonik_core_htcmock_test_client:$env:VERSION
timeout-minutes: 10
shell: powershell

- name: Run HtcMock test 100 tasks 1 level
timeout-minutes: 3
shell: powershell
Expand Down Expand Up @@ -724,7 +751,7 @@ jobs:
name: "Test connectivity - Ca installed ${{ matrix.ca }}"
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true
Expand All @@ -739,6 +766,8 @@ jobs:
- name: Setup Terraform
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
with:
terraform_version: latest

- name: Deploy Core
run: |
Expand Down Expand Up @@ -808,7 +837,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true
Expand All @@ -823,6 +852,8 @@ jobs:
- name: Setup Terraform
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
with:
terraform_version: latest

- name: Deploy Core
run: |
Expand Down Expand Up @@ -926,7 +957,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true
Expand All @@ -941,6 +972,8 @@ jobs:
- name: Setup Terraform
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
with:
terraform_version: latest

- name: Deploy Core
run: |
Expand Down Expand Up @@ -998,7 +1031,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true
Expand All @@ -1013,6 +1046,8 @@ jobs:
- name: Setup Terraform
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
with:
terraform_version: latest

- name: Deploy Core
run: |
Expand Down Expand Up @@ -1068,7 +1103,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ github.ref }}
submodules: true
Expand All @@ -1083,6 +1118,8 @@ jobs:
- name: Setup Terraform
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
with:
terraform_version: latest

- name: Deploy Core
run: |
Expand Down
Loading

0 comments on commit 8b83d78

Please sign in to comment.