diff --git a/.docs/content/0.installation/5.version-migration.md b/.docs/content/0.installation/5.version-migration.md new file mode 100644 index 000000000..c8b25674a --- /dev/null +++ b/.docs/content/0.installation/5.version-migration.md @@ -0,0 +1,51 @@ +# How to migrate ArmoniK.Core dependencies during upgrade ? + +## 0.28.x -> 0.29.x + +### Database + +This version changes the structure of a Result in the database. It introduces a new field called `OpaqueId` which holds the identifier of its associated value in the Object Storage. Previously, the ResultId was used. The following MongoDB request converts the ResultId into the OpaqueId to support the new implementation. + +```js +db.Result.updateMany({}, + [{ + $addFields: { + OpaqueId: { + $function: { + body: function(data) { + const base64Chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + const bytes = []; + for (let i = 0; i < data.length; i++) { + bytes.push(data.charCodeAt(i)); + } + + let base64 = ''; + let i = 0; + while (i < bytes.length) { + let byte1 = bytes[i++] || 0; + let byte2 = bytes[i++] || 0; + let byte3 = bytes[i++] || 0; + + let enc1 = byte1 >> 2; + let enc2 = ((byte1 & 3) << 4) | (byte2 >> 4); + let enc3 = ((byte2 & 15) << 2) | (byte3 >> 6); + let enc4 = byte3 & 63; + + if (isNaN(byte2)) { + enc3 = enc4 = 64; + } else if (isNaN(byte3)) { + enc4 = 64; + } + + base64 += base64Chars[enc1] + base64Chars[enc2] + base64Chars[enc3] + base64Chars[enc4]; + } + + return BinData(0, base64); + }, + args: ["$_id"], + lang: "js" + } + } + } + }]) +``` diff --git a/.docs/package.json b/.docs/package.json index 4f0568867..b1fd147a8 100644 --- a/.docs/package.json +++ b/.docs/package.json @@ -15,7 +15,7 @@ }, "dependencies": { "@aneoconsultingfr/armonik-docs-theme": "^0.6.12", - "mermaid": "^10.2.3", + "mermaid": "^11.4.0", "nuxt": "^3.6.1" }, "devDependencies": { diff --git a/.docs/pnpm-lock.yaml b/.docs/pnpm-lock.yaml index 4eb3257be..828465e15 100644 --- a/.docs/pnpm-lock.yaml +++ b/.docs/pnpm-lock.yaml @@ -10,17 +10,17 @@ importers: dependencies: '@aneoconsultingfr/armonik-docs-theme': specifier: ^0.6.12 - version: 0.6.12(@types/node@18.15.5)(@unhead/vue@1.1.28(vue@3.3.4))(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(postcss@8.4.24)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6))(vue@3.3.4) + version: 0.6.12(@types/node@18.15.5)(@unhead/vue@1.1.28(vue@3.3.4))(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(postcss@8.4.24)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6))(vue@3.3.4) mermaid: - specifier: ^10.2.3 - version: 10.2.3 + specifier: ^11.4.0 + version: 11.4.0 nuxt: specifier: ^3.6.1 - version: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4) + version: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5) devDependencies: '@nuxt/eslint-config': specifier: ^0.3.8 - version: 0.3.8(eslint@9.0.0)(typescript@5.0.4) + version: 0.3.8(eslint@9.0.0)(typescript@4.9.5) case-police: specifier: ^0.6.1 version: 0.6.1 @@ -44,6 +44,12 @@ packages: '@aneoconsultingfr/armonik-docs-theme@0.6.12': resolution: {integrity: sha512-jvurHffr7TukP7GuCQKnHt3lSPQVviZ3ROBMI9hz1T7Hxh5wPyi+sOSbkl8HolUe6frbCaGCRgKm02VyTF3EKw==} + '@antfu/install-pkg@0.4.1': + resolution: {integrity: sha512-T7yB5QNG29afhWVkVq7XeIMBa5U/vs9mX69YqayXypPRmYzUmzwnYltplHmPtZ4HPCn+sQKeXW8I47wCbuBOjw==} + + '@antfu/utils@0.7.10': + resolution: {integrity: sha512-+562v9k4aI80m1+VuMHehNJWLOFjBnXn3tdOitzD0il5b7smkSBal4+a3oKiQTbrwMmN/TBUMDvbdoWDehgOww==} + '@antfu/utils@0.7.4': resolution: {integrity: sha512-qe8Nmh9rYI/HIspLSTwtbMFPj6dISG6+dJnOguTlPNXtCvS2uezdxscVBb7/3DrmNbQK49TDqpkSQ1chbRGdpQ==} @@ -207,8 +213,26 @@ packages: resolution: {integrity: sha512-zo3MIHGOkPOfoRXitsgHLjEXmlDaD/5KU1Uzuc9GNiZPhSqVxVRtxuPaSBZDsYZ9qV88AjtMtWW7ww98loJ9KA==} engines: {node: '>=6.9.0'} - '@braintree/sanitize-url@6.0.2': - resolution: {integrity: sha512-Tbsj02wXCbqGmzdnXNk0SOF19ChhRU70BsroIi4Pm6Ehp56in6vch94mfbdQ17DozxkL3BAVjbZ4Qc1a0HFRAg==} + '@braintree/sanitize-url@6.0.4': + resolution: {integrity: sha512-s3jaWicZd0pkP0jf5ysyHUI/RE7MHos6qlToFcGWXVp+ykHOy77OUMrfbgJ9it2C5bow7OIQwYYaHjk9XlBQ2A==} + + '@braintree/sanitize-url@7.1.0': + resolution: {integrity: sha512-o+UlMLt49RvtCASlOMW0AkHnabN9wR9rwCCherxO0yG4Npy34GkvrAqdXQvrhNs+jh+gkK8gB8Lf05qL/O7KWg==} + + '@chevrotain/cst-dts-gen@11.0.3': + resolution: {integrity: sha512-BvIKpRLeS/8UbfxXxgC33xOumsacaeCKAjAeLyOn7Pcp95HiRbrpl14S+9vaZLolnbssPIUuiUd8IvgkRyt6NQ==} + + '@chevrotain/gast@11.0.3': + resolution: {integrity: sha512-+qNfcoNk70PyS/uxmj3li5NiECO+2YKZZQMbmjTqRI3Qchu8Hig/Q9vgkHpI3alNjr7M+a2St5pw5w5F6NL5/Q==} + + '@chevrotain/regexp-to-ast@11.0.3': + resolution: {integrity: sha512-1fMHaBZxLFvWI067AVbGJav1eRY7N8DDvYCTwGBiE/ytKBgP8azTdgyrKyWZ9Mfh09eHWb5PgTSO8wi7U824RA==} + + '@chevrotain/types@11.0.3': + resolution: {integrity: sha512-gsiM3G8b58kZC2HaWR50gu6Y1440cHiJ+i3JUvcp/35JchYejb2+5MVeJK0iKThYpAa/P2PYFV4hoi44HD+aHQ==} + + '@chevrotain/utils@11.0.3': + resolution: {integrity: sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ==} '@cloudflare/kv-asset-handler@0.3.0': resolution: {integrity: sha512-9CB/MKf/wdvbfkUdfrj+OkEwZ5b7rws0eogJ4293h+7b6KX5toPwym+VQKmILafNB9YiehqY0DlNrDcDhdWHSQ==} @@ -236,12 +260,15 @@ packages: '@esbuild-kit/cjs-loader@2.4.2': resolution: {integrity: sha512-BDXFbYOJzT/NBEtp71cvsrGPwGAMGRB/349rwKuoxNSiKjPraNNnlK6MIIabViCjqZugu6j+xeMDlEkWdHHJSg==} + deprecated: 'Merged into tsx: https://tsx.is' '@esbuild-kit/core-utils@3.1.0': resolution: {integrity: sha512-Uuk8RpCg/7fdHSceR1M6XbSZFSuMrxcePFuGgyvsBn+u339dk5OeL4jv2EojwTN2st/unJGsVm4qHWjWNmJ/tw==} + deprecated: 'Merged into tsx: https://tsx.is' '@esbuild-kit/esm-loader@2.5.5': resolution: {integrity: sha512-Qwfvj/qoPbClxCRNuac1Du01r9gvNOT+pMYtJDapfB1eoGN1YlJ1BixLyL9WVENRx5RXgNLdfYdx/CuswlGhMw==} + deprecated: 'Merged into tsx: https://tsx.is' '@esbuild/android-arm64@0.17.19': resolution: {integrity: sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==} @@ -541,6 +568,7 @@ packages: '@humanwhocodes/config-array@0.12.3': resolution: {integrity: sha512-jsNnTBlMWuTpDkeE3on7+dWJi0D6fdDfeANj/w7MpS8ztROCoLvIO2nG0CcFj+E4k8j4QrSTh4Oryi3i2G669g==} engines: {node: '>=10.10.0'} + deprecated: Use @eslint/config-array instead '@humanwhocodes/module-importer@1.0.1': resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} @@ -548,10 +576,14 @@ packages: '@humanwhocodes/object-schema@2.0.3': resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} + deprecated: Use @eslint/object-schema instead '@iconify/types@2.0.0': resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==} + '@iconify/utils@2.1.33': + resolution: {integrity: sha512-jP9h6v/g0BIZx0p7XGJJVtkVnydtbgTgt9mVNcGDYwaa7UhdHdI9dvoq+gKj9sijMSJKxUPEG2JyjsgXjxL7Kw==} + '@iconify/vue@4.1.0': resolution: {integrity: sha512-rBQVxNoSDooqgWkQg2MqkIHkH/huNuvXGqui5wijc1zLnU7TKzbBHW9VGmbnV4asNTmIHmqV4Nvt0M2rZ/9nHA==} peerDependencies: @@ -593,6 +625,9 @@ packages: resolution: {integrity: sha512-4ySo4CjzStuprMwk35H5pPbkymjv1SF3jGLj6rAHp/xT/RF7TL7bd9CTm1xDY49K2qF7jmR/g7k+SkLETP6opA==} hasBin: true + '@mermaid-js/parser@0.3.0': + resolution: {integrity: sha512-HsvL6zgE5sUPGgkIDlmAWR1HTNHz2Iy11BAWPTa4Jjabkpguy4Ze2gzfLrg6pdRuBvFwgUYyxiaNqZwrEEXepA==} + '@netlify/functions@1.6.0': resolution: {integrity: sha512-6G92AlcpFrQG72XU8YH8pg94eDnq7+Q0YJhb8x4qNpdGsvuzvrfHWBmqFGp/Yshmv4wex9lpsTRZOocdrA2erQ==} engines: {node: '>=14.0.0'} @@ -918,8 +953,104 @@ packages: resolution: {integrity: sha512-qaGV9ltJP0EO25YfFUPhxRVK0evXFIAGicsVXuRim4Ed9cjPxYhNnNJ49SFmbeLgtxpslIkX317IgpfcHPVj/A==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} - '@types/debug@4.1.7': - resolution: {integrity: sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==} + '@types/d3-array@3.2.1': + resolution: {integrity: sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==} + + '@types/d3-axis@3.0.6': + resolution: {integrity: sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==} + + '@types/d3-brush@3.0.6': + resolution: {integrity: sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==} + + '@types/d3-chord@3.0.6': + resolution: {integrity: sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==} + + '@types/d3-color@3.1.3': + resolution: {integrity: sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==} + + '@types/d3-contour@3.0.6': + resolution: {integrity: sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==} + + '@types/d3-delaunay@6.0.4': + resolution: {integrity: sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==} + + '@types/d3-dispatch@3.0.6': + resolution: {integrity: sha512-4fvZhzMeeuBJYZXRXrRIQnvUYfyXwYmLsdiN7XXmVNQKKw1cM8a5WdID0g1hVFZDqT9ZqZEY5pD44p24VS7iZQ==} + + '@types/d3-drag@3.0.7': + resolution: {integrity: sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==} + + '@types/d3-dsv@3.0.7': + resolution: {integrity: sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==} + + '@types/d3-ease@3.0.2': + resolution: {integrity: sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==} + + '@types/d3-fetch@3.0.7': + resolution: {integrity: sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==} + + '@types/d3-force@3.0.10': + resolution: {integrity: sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==} + + '@types/d3-format@3.0.4': + resolution: {integrity: sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==} + + '@types/d3-geo@3.1.0': + resolution: {integrity: sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==} + + '@types/d3-hierarchy@3.1.7': + resolution: {integrity: sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==} + + '@types/d3-interpolate@3.0.4': + resolution: {integrity: sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==} + + '@types/d3-path@3.1.0': + resolution: {integrity: sha512-P2dlU/q51fkOc/Gfl3Ul9kicV7l+ra934qBFXCFhrZMOL6du1TM0pm1ThYvENukyOn5h9v+yMJ9Fn5JK4QozrQ==} + + '@types/d3-polygon@3.0.2': + resolution: {integrity: sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==} + + '@types/d3-quadtree@3.0.6': + resolution: {integrity: sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==} + + '@types/d3-random@3.0.3': + resolution: {integrity: sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ==} + + '@types/d3-scale-chromatic@3.0.3': + resolution: {integrity: sha512-laXM4+1o5ImZv3RpFAsTRn3TEkzqkytiOY0Dz0sq5cnd1dtNlk6sHLon4OvqaiJb28T0S/TdsBI3Sjsy+keJrw==} + + '@types/d3-scale@4.0.8': + resolution: {integrity: sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ==} + + '@types/d3-selection@3.0.11': + resolution: {integrity: sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==} + + '@types/d3-shape@3.1.6': + resolution: {integrity: sha512-5KKk5aKGu2I+O6SONMYSNflgiP0WfZIQvVUMan50wHsLG1G94JlxEVnCpQARfTtzytuY0p/9PXXZb3I7giofIA==} + + '@types/d3-time-format@4.0.3': + resolution: {integrity: sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==} + + '@types/d3-time@3.0.3': + resolution: {integrity: sha512-2p6olUZ4w3s+07q3Tm2dbiMZy5pCDfYwtLXXHUnVzXgQlZ/OyPtUz6OL382BkOuGlLXqfT+wqv8Fw2v8/0geBw==} + + '@types/d3-timer@3.0.2': + resolution: {integrity: sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==} + + '@types/d3-transition@3.0.9': + resolution: {integrity: sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==} + + '@types/d3-zoom@3.0.8': + resolution: {integrity: sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==} + + '@types/d3@7.4.3': + resolution: {integrity: sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==} + + '@types/debug@4.1.12': + resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} + + '@types/dompurify@3.0.5': + resolution: {integrity: sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==} '@types/eslint@8.56.9': resolution: {integrity: sha512-W4W3KcqzjJ0sHg2vAq9vfml6OhsJ53TcUjUqfzzZf/EChUtwspszj/S0pzMxnfRcO55/iGq47dscXw71Fxc4Zg==} @@ -930,6 +1061,9 @@ packages: '@types/fs-extra@11.0.1': resolution: {integrity: sha512-MxObHvNl4A69ofaTRU8DFqvgzzv8s9yRtaPPm5gud9HDNvpB3GPQFvNuTWAI59B9huVGV5jXYJwbCsmBsOGYWA==} + '@types/geojson@7946.0.14': + resolution: {integrity: sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg==} + '@types/hast@2.3.4': resolution: {integrity: sha512-wLEm0QvaoawEDoTRwzTXp4b4jpwiJDvR5KMnFnVodm3scufTlBOWRD6N1OBf9TZMhjlNsSfcO5V+7AF4+Vy+9g==} @@ -945,11 +1079,11 @@ packages: '@types/jsonfile@6.1.1': resolution: {integrity: sha512-GSgiRCVeapDN+3pqA35IkQwasaCh/0YFH5dEF6S88iDvEn901DjOeH3/QPY+XYP1DFzDZPvIvfeEgk+7br5png==} - '@types/mdast@3.0.10': - resolution: {integrity: sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==} + '@types/mdast@3.0.15': + resolution: {integrity: sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==} - '@types/ms@0.7.31': - resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} + '@types/ms@0.7.34': + resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} '@types/node@17.0.45': resolution: {integrity: sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==} @@ -975,8 +1109,14 @@ packages: '@types/semver@7.5.8': resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==} - '@types/unist@2.0.6': - resolution: {integrity: sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==} + '@types/trusted-types@2.0.7': + resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} + + '@types/unist@2.0.11': + resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} + + '@types/unist@3.0.3': + resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} '@types/web-bluetooth@0.0.16': resolution: {integrity: sha512-oh8q2Zc32S6gd/j50GowEjKLoOVOwHP/bWVjKJInBwQqdOYMdPrf1oVlelTlyfFK3CKxL1uahMDAr+vy8T7yMQ==} @@ -1226,6 +1366,11 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + acorn@8.14.0: + resolution: {integrity: sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==} + engines: {node: '>=0.4.0'} + hasBin: true + acorn@8.9.0: resolution: {integrity: sha512-jaVNAFBHNLXspO543WnNNPZFRtavh3skAkITqD0/2aeMkKZTN+254PyhwxFYrk3vQ1xfY+2wbesJMs/JC8/PwQ==} engines: {node: '>=0.4.0'} @@ -1306,10 +1451,12 @@ packages: are-we-there-yet@2.0.0: resolution: {integrity: sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==} engines: {node: '>=10'} + deprecated: This package is no longer supported. are-we-there-yet@3.0.1: resolution: {integrity: sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + deprecated: This package is no longer supported. arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} @@ -1548,6 +1695,14 @@ packages: resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} engines: {node: '>= 6'} + chevrotain-allstar@0.3.1: + resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==} + peerDependencies: + chevrotain: ^11.0.0 + + chevrotain@11.0.3: + resolution: {integrity: sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw==} + chokidar@3.5.3: resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==} engines: {node: '>= 8.10.0'} @@ -1681,6 +1836,9 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + confbox@0.1.8: + resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} + consola@2.15.3: resolution: {integrity: sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==} @@ -1804,12 +1962,15 @@ packages: peerDependencies: cytoscape: ^3.2.0 - cytoscape@3.23.0: - resolution: {integrity: sha512-gRZqJj/1kiAVPkrVFvz/GccxsXhF3Qwpptl32gKKypO4IlqnKBjTOu+HbXtEggSGzC5KCaHp3/F7GgENrtsFkA==} + cytoscape@3.30.3: + resolution: {integrity: sha512-HncJ9gGJbVtw7YXtIs3+6YAFSSiKsom0amWc33Z7QbylbY2JGMrA0yz4EwrdTScZxnwclXeEZHzO5pxoy0ZE4g==} engines: {node: '>=0.10'} - d3-array@3.2.2: - resolution: {integrity: sha512-yEEyEAbDrF8C6Ob2myOBLjwBLck1Z89jMGFee0oPsn95GqjerpaOA4ch+vc2l0FNFFwMD5N7OCSEN5eAlsUbgQ==} + d3-array@2.12.1: + resolution: {integrity: sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==} + + d3-array@3.2.4: + resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} engines: {node: '>=12'} d3-axis@3.0.0: @@ -1832,8 +1993,8 @@ packages: resolution: {integrity: sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==} engines: {node: '>=12'} - d3-delaunay@6.0.2: - resolution: {integrity: sha512-IMLNldruDQScrcfT+MWnazhHbDJhcRJyOEBAJfwQnHle1RPh6WDuLvxNArUju2VSMSUuKlY5BGHRJ2cYyoFLQQ==} + d3-delaunay@6.0.4: + resolution: {integrity: sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==} engines: {node: '>=12'} d3-dispatch@3.0.1: @@ -1865,8 +2026,8 @@ packages: resolution: {integrity: sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==} engines: {node: '>=12'} - d3-geo@3.1.0: - resolution: {integrity: sha512-JEo5HxXDdDYXCaWdwLRt79y7giK8SbhZJbFWXqbRTolCHFI5jRqteLzCsq51NKbUoX0PjBVSohxrx+NoOUujYA==} + d3-geo@3.1.1: + resolution: {integrity: sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==} engines: {node: '>=12'} d3-hierarchy@3.1.2: @@ -1877,6 +2038,9 @@ packages: resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} engines: {node: '>=12'} + d3-path@1.0.9: + resolution: {integrity: sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==} + d3-path@3.1.0: resolution: {integrity: sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==} engines: {node: '>=12'} @@ -1893,8 +2057,11 @@ packages: resolution: {integrity: sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==} engines: {node: '>=12'} - d3-scale-chromatic@3.0.0: - resolution: {integrity: sha512-Lx9thtxAKrO2Pq6OO2Ua474opeziKr279P/TKZsMAhYyNDD3EnCffdbgeSYN5O7m2ByQsxtuP2CSDczNUIZ22g==} + d3-sankey@0.12.3: + resolution: {integrity: sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==} + + d3-scale-chromatic@3.1.0: + resolution: {integrity: sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==} engines: {node: '>=12'} d3-scale@4.0.2: @@ -1905,6 +2072,9 @@ packages: resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} engines: {node: '>=12'} + d3-shape@1.3.7: + resolution: {integrity: sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==} + d3-shape@3.2.0: resolution: {integrity: sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==} engines: {node: '>=12'} @@ -1931,19 +2101,22 @@ packages: resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} engines: {node: '>=12'} - d3@7.8.2: - resolution: {integrity: sha512-WXty7qOGSHb7HR7CfOzwN1Gw04MUOzN8qh9ZUsvwycIMb4DYMpY9xczZ6jUorGtO6bR9BPMPaueIKwiDxu9uiQ==} + d3@7.9.0: + resolution: {integrity: sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==} engines: {node: '>=12'} dagre-d3-es@7.0.10: resolution: {integrity: sha512-qTCQmEhcynucuaZgY5/+ti3X/rnszKZhEQH/ZdWdtP1tA/y3VoHJzcVrO9pjjJCNpigfscAtoUB5ONcd2wNn0A==} + dagre-d3-es@7.0.11: + resolution: {integrity: sha512-tvlJLyQf834SylNKax8Wkzco/1ias1OPw8DcUMDE7oUIoSEW25riQVuiu/0OWEFqT0cxHT3Pa9/D82Jr47IONw==} + data-uri-to-buffer@4.0.1: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} - dayjs@1.11.7: - resolution: {integrity: sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==} + dayjs@1.11.13: + resolution: {integrity: sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==} de-indent@1.0.2: resolution: {integrity: sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==} @@ -1973,6 +2146,15 @@ packages: supports-color: optional: true + debug@4.3.7: + resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + decode-named-character-reference@1.0.2: resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==} @@ -2033,8 +2215,8 @@ packages: defu@6.1.2: resolution: {integrity: sha512-+uO4+qr7msjNNWKYPHqN/3+Dx3NFkmIzayk2L1MyZQlvgZb/J1A0fo410dpKrN2SnqFjt8n4JL8fDJE0wIgjFQ==} - delaunator@5.0.0: - resolution: {integrity: sha512-AyLvtyJdbv/U1GkiS6gUUzclRoAY4Gs75qkMygJJhU75LW4DNuSF2RMzpxs9jw9Oz1BobHjTdkG3zdP55VxAqw==} + delaunator@5.0.1: + resolution: {integrity: sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw==} delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} @@ -2079,6 +2261,10 @@ packages: resolution: {integrity: sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==} engines: {node: '>=0.3.1'} + diff@5.2.0: + resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==} + engines: {node: '>=0.3.1'} + dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} @@ -2097,8 +2283,8 @@ packages: resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} engines: {node: '>= 4'} - dompurify@3.0.3: - resolution: {integrity: sha512-axQ9zieHLnAnHh0sfAamKYiqXMJAVwu+LM/alQ7WDagoWessyWvMSFyW65CqF3owufNu8HBcE4cM2Vflu7YWcQ==} + dompurify@3.1.6: + resolution: {integrity: sha512-cTOAhc36AalkjtBpfG6O8JimdTMWNXjiePT2xQH/ppBGi/4uIpmj8eKyIkMJErXWARyINV/sB38yf8JCLF5pbQ==} domutils@3.0.1: resolution: {integrity: sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==} @@ -2133,8 +2319,8 @@ packages: electron-to-chromium@1.4.740: resolution: {integrity: sha512-Yvg5i+iyv7Xm18BRdVPVm8lc7kgxM3r6iwqCH2zB7QZy1kZRNmd0Zqm0zcD9XoFREE5/5rwIuIAOT+/mzGcnZg==} - elkjs@0.8.2: - resolution: {integrity: sha512-L6uRgvZTH+4OF5NE/MBbzQx/WYpru1xCBE9respNj6qznEewGUIfhzmm7horWWxbNO2M0WckQypGctR8lH79xQ==} + elkjs@0.9.3: + resolution: {integrity: sha512-f/ZeWvW/BCXbhGEf1Ujp29EASo/lk1FDnETgNKwJrsVvGZhUWCZyg3xLJjAsxfOmt8KjswHmI5EwCQcPMpOYhQ==} emoji-regex@10.2.1: resolution: {integrity: sha512-97g6QgOk8zlDRdgq1WxwgTMgEWGVAQvB5Fdpgc1MkNy56la5SKP9GsMXKDOdqwn90/41a8yPwIGk1Y6WVbeMQA==} @@ -2507,10 +2693,12 @@ packages: gauge@3.0.2: resolution: {integrity: sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==} engines: {node: '>=10'} + deprecated: This package is no longer supported. gauge@4.0.4: resolution: {integrity: sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + deprecated: This package is no longer supported. gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} @@ -2576,10 +2764,12 @@ packages: glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + deprecated: Glob versions prior to v9 are no longer supported glob@8.1.0: resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} engines: {node: '>=12'} + deprecated: Glob versions prior to v9 are no longer supported global-dirs@3.0.1: resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==} @@ -2629,6 +2819,9 @@ packages: h3@1.7.0: resolution: {integrity: sha512-iJJz2Pn2rC0j8CB3rkFMs0K269W7hDVOC7eL3qne5Joy4JZX1W7id7PBFV593GboHDOx0PzgO6ocqsynrIvdxw==} + hachure-fill@0.5.2: + resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==} + has-flag@3.0.0: resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} engines: {node: '>=4'} @@ -2696,9 +2889,6 @@ packages: header-case@2.0.4: resolution: {integrity: sha512-H/vuk5TEEVZwrR0lp2zed9OCo1uAILMlx0JEMgC26rzyJJ3N1v6XkwHHXJQdR2doSjcGPM6OKPYoJgf0plJ11Q==} - heap@0.2.7: - resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==} - hookable@5.5.3: resolution: {integrity: sha512-Yc+BQe8SvoXH1643Qez1zqLRmbA5rCL+sSmk6TVos0LWVfNIB7PGncdlId77WzLGSIB5KaWgTaNTs2lNVEI6VQ==} @@ -2805,6 +2995,7 @@ packages: inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -2824,6 +3015,9 @@ packages: resolution: {integrity: sha512-VV2ZOZe4ilLlOgEo7drIdzbi+EYJcNty0leF2vJq49zOW8+IoK1miJ+V5FjZY/X21Io29j66T/AiqHvS4tPIrw==} engines: {node: '>=14.18.0'} + internmap@1.0.1: + resolution: {integrity: sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==} + internmap@2.0.3: resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} engines: {node: '>=12'} @@ -3071,11 +3265,15 @@ packages: resolution: {integrity: sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==} engines: {'0': node >= 0.2.0} + katex@0.16.11: + resolution: {integrity: sha512-RQrI8rlHY92OLf3rho/Ts8i/XvjgguEjOkO1BEXcU3N8BqPpSzBNwV/G0Ukr+P/l3ivvJUE/Fa/CwbS6HesGNQ==} + hasBin: true + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} - khroma@2.0.0: - resolution: {integrity: sha512-2J8rDNlQWbtiNYThZRvmMv5yt44ZakX+Tz5ZIp/mN1pt4snn+m030Va5Z4v8xA0cQFDXBwO/8i42xL4QPsVk3g==} + khroma@2.1.0: + resolution: {integrity: sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==} kleur@3.0.3: resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} @@ -3095,6 +3293,10 @@ packages: kolorist@1.8.0: resolution: {integrity: sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==} + langium@3.0.0: + resolution: {integrity: sha512-+Ez9EoiByeoTu/2BXmEaZ06iPNXM6thWJp02KfBO/raSMyCJ4jw7AkWWa+zBCTm0+Tw1Fj9FOxdqSskyN5nAwg==} + engines: {node: '>=16.0.0'} + launch-editor@2.6.0: resolution: {integrity: sha512-JpDCcQnyAAzZZaZ7vEiSqL690w7dAEyLao+KC96zBplnYbJS7TYNjvM3M7y3dGz+v7aIsJk3hllWuc0kWAjyRQ==} @@ -3135,6 +3337,10 @@ packages: resolution: {integrity: sha512-SFppqq5p42fe2qcZQqqEOiVRXl+WCP1MdT6k7BDEW1j++sp5fIY+/fdRQitvKgB5BrBcmrs5m/L0v2FrU5MY1g==} engines: {node: '>=14'} + local-pkg@0.5.0: + resolution: {integrity: sha512-ok6z3qlYyCDS4ZEU27HaU6x/xZa9Whf8jD4ptH5UZTQYZVYeb9bnZ3ojVhiJNLiXK1Hfc0GNbLXcmZ5plLDDBg==} + engines: {node: '>=14'} + locate-path@5.0.0: resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} engines: {node: '>=8'} @@ -3279,6 +3485,11 @@ packages: resolution: {integrity: sha512-ASAzqpODstu/Qsk0xW5BPgWnK/qjpBQ4e7IpsSvvFXcfYIjanLTdwFRJK1SIEEh0fGSMKXcJf/qhaZYHyME0wA==} engines: {node: '>=16'} + marked@13.0.3: + resolution: {integrity: sha512-rqRix3/TWzE9rIoFGIn8JmsVfhiuC8VIQ8IdX5TfzmeBucdY05/0UlzKaw0eVtpcN/OdVFpBk7CjKGo9iHJ/zA==} + engines: {node: '>= 18'} + hasBin: true + marky@1.2.5: resolution: {integrity: sha512-q9JtQJKjpsVxCRVgQ+WapguSbKC3SQ5HEzFGPAJMStgh3QjCawp00UKv3MTTAArTmGmmPUvllHZoNbZ3gs0I+Q==} @@ -3291,8 +3502,8 @@ packages: mdast-util-find-and-replace@2.2.2: resolution: {integrity: sha512-MTtdFRz/eMDHXzeK6W3dO7mXUlF82Gom4y0oOgvHhh/HXZAGvIQDUvQ0SuUx+j2tv44b8xTHOm8K/9OoRFnXKw==} - mdast-util-from-markdown@1.3.0: - resolution: {integrity: sha512-HN3W1gRIuN/ZW295c7zi7g9lVBllMgZE40RxCX37wrTPWXCWtpvOZdfnuK+1WNpvZje6XuJeI3Wnb4TJEUem+g==} + mdast-util-from-markdown@1.3.1: + resolution: {integrity: sha512-4xTO/M8c82qBcnQc1tgpNtubGUW/Y1tBQ1B0i5CtSoelOLKFYlElIr3bvgREYYO5iRqbMY1YuqZng0GVOI8Qww==} mdast-util-gfm-autolink-literal@1.0.3: resolution: {integrity: sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==} @@ -3321,8 +3532,8 @@ packages: mdast-util-to-markdown@1.5.0: resolution: {integrity: sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A==} - mdast-util-to-string@3.1.1: - resolution: {integrity: sha512-tGvhT94e+cVnQt8JWE9/b3cUQZWS732TJxXHktvP+BYo62PpYD53Ls/6cC60rW21dW+txxiM4zMdc6abASvZKA==} + mdast-util-to-string@3.2.0: + resolution: {integrity: sha512-V4Zn/ncyN1QNSqSBxTrMOLpjr+IKdHl2v3KVLoWmDPscP4r9GcCi71gjgvUV1SFSKh92AjAG4peFuBl2/YgCJg==} mdn-data@2.0.28: resolution: {integrity: sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==} @@ -3344,11 +3555,14 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} - mermaid@10.2.3: - resolution: {integrity: sha512-cMVE5s9PlQvOwfORkyVpr5beMsLdInrycAosdr+tpZ0WFjG4RJ/bUHST7aTgHNJbujHkdBRAm+N50P3puQOfPw==} + mermaid@10.9.3: + resolution: {integrity: sha512-V80X1isSEvAewIL3xhmz/rVmc27CVljcsbWxkxlWJWY/1kQa4XOABqpDl2qQLGKzpKm6WbTfUEKImBlUfFYArw==} - micromark-core-commonmark@1.0.6: - resolution: {integrity: sha512-K+PkJTxqjFfSNkfAhp4GB+cZPfQd6dxtTXnf+RjZOV7T4EEXnvgzOcnp+eSTmpGk9d1S9sL6/lqrgSNn/s0HZA==} + mermaid@11.4.0: + resolution: {integrity: sha512-mxCfEYvADJqOiHfGpJXLs4/fAjHz448rH0pfY5fAoxiz70rQiDSzUUy4dNET2T08i46IVpjohPd6WWbzmRHiPA==} + + micromark-core-commonmark@1.1.0: + resolution: {integrity: sha512-BgHO1aRbolh2hcrzL2d1La37V0Aoz73ymF8rAcKnohLy93titmv62E0gP8Hrx9PKcKrqCZ1BbLGbP3bEhoXYlw==} micromark-extension-gfm-autolink-literal@1.0.3: resolution: {integrity: sha512-i3dmvU0htawfWED8aHMMAzAVp/F0Z+0bPh3YrbTPPL1v4YAlCZpy5rBO5p0LPYiZo0zFVkoYh7vDU7yQSiCMjg==} @@ -3371,65 +3585,68 @@ packages: micromark-extension-gfm@2.0.1: resolution: {integrity: sha512-p2sGjajLa0iYiGQdT0oelahRYtMWvLjy8J9LOCxzIQsllMCGLbsLW+Nc+N4vi02jcRJvedVJ68cjelKIO6bpDA==} - micromark-factory-destination@1.0.0: - resolution: {integrity: sha512-eUBA7Rs1/xtTVun9TmV3gjfPz2wEwgK5R5xcbIM5ZYAtvGF6JkyaDsj0agx8urXnO31tEO6Ug83iVH3tdedLnw==} + micromark-factory-destination@1.1.0: + resolution: {integrity: sha512-XaNDROBgx9SgSChd69pjiGKbV+nfHGDPVYFs5dOoDd7ZnMAE+Cuu91BCpsY8RT2NP9vo/B8pds2VQNCLiu0zhg==} - micromark-factory-label@1.0.2: - resolution: {integrity: sha512-CTIwxlOnU7dEshXDQ+dsr2n+yxpP0+fn271pu0bwDIS8uqfFcumXpj5mLn3hSC8iw2MUr6Gx8EcKng1dD7i6hg==} + micromark-factory-label@1.1.0: + resolution: {integrity: sha512-OLtyez4vZo/1NjxGhcpDSbHQ+m0IIGnT8BoPamh+7jVlzLJBH98zzuCoUeMxvM6WsNeh8wx8cKvqLiPHEACn0w==} - micromark-factory-space@1.0.0: - resolution: {integrity: sha512-qUmqs4kj9a5yBnk3JMLyjtWYN6Mzfcx8uJfi5XAveBniDevmZasdGBba5b4QsvRcAkmvGo5ACmSUmyGiKTLZew==} + micromark-factory-space@1.1.0: + resolution: {integrity: sha512-cRzEj7c0OL4Mw2v6nwzttyOZe8XY/Z8G0rzmWQZTBi/jjwyw/U4uqKtUORXQrR5bAZZnbTI/feRV/R7hc4jQYQ==} - micromark-factory-title@1.0.2: - resolution: {integrity: sha512-zily+Nr4yFqgMGRKLpTVsNl5L4PMu485fGFDOQJQBl2NFpjGte1e86zC0da93wf97jrc4+2G2GQudFMHn3IX+A==} + micromark-factory-title@1.1.0: + resolution: {integrity: sha512-J7n9R3vMmgjDOCY8NPw55jiyaQnH5kBdV2/UXCtZIpnHH3P6nHUKaH7XXEYuWwx/xUJcawa8plLBEjMPU24HzQ==} micromark-factory-whitespace@1.0.0: resolution: {integrity: sha512-Qx7uEyahU1lt1RnsECBiuEbfr9INjQTGa6Err+gF3g0Tx4YEviPbqqGKNv/NrBaE7dVHdn1bVZKM/n5I/Bak7A==} - micromark-util-character@1.1.0: - resolution: {integrity: sha512-agJ5B3unGNJ9rJvADMJ5ZiYjBRyDpzKAOk01Kpi1TKhlT1APx3XZk6eN7RtSz1erbWHC2L8T3xLZ81wdtGRZzg==} + micromark-factory-whitespace@1.1.0: + resolution: {integrity: sha512-v2WlmiymVSp5oMg+1Q0N1Lxmt6pMhIHD457whWM7/GUlEks1hI9xj5w3zbc4uuMKXGisksZk8DzP2UyGbGqNsQ==} + + micromark-util-character@1.2.0: + resolution: {integrity: sha512-lXraTwcX3yH/vMDaFWCQJP1uIszLVebzUa3ZHdrgxr7KEU/9mL4mVgCpGbyhvNLNlauROiNUq7WN5u7ndbY6xg==} - micromark-util-chunked@1.0.0: - resolution: {integrity: sha512-5e8xTis5tEZKgesfbQMKRCyzvffRRUX+lK/y+DvsMFdabAicPkkZV6gO+FEWi9RfuKKoxxPwNL+dFF0SMImc1g==} + micromark-util-chunked@1.1.0: + resolution: {integrity: sha512-Ye01HXpkZPNcV6FiyoW2fGZDUw4Yc7vT0E9Sad83+bEDiCJ1uXu0S3mr8WLpsz3HaG3x2q0HM6CTuPdcZcluFQ==} - micromark-util-classify-character@1.0.0: - resolution: {integrity: sha512-F8oW2KKrQRb3vS5ud5HIqBVkCqQi224Nm55o5wYLzY/9PwHGXC01tr3d7+TqHHz6zrKQ72Okwtvm/xQm6OVNZA==} + micromark-util-classify-character@1.1.0: + resolution: {integrity: sha512-SL0wLxtKSnklKSUplok1WQFoGhUdWYKggKUiqhX+Swala+BtptGCu5iPRc+xvzJ4PXE/hwM3FNXsfEVgoZsWbw==} - micromark-util-combine-extensions@1.0.0: - resolution: {integrity: sha512-J8H058vFBdo/6+AsjHp2NF7AJ02SZtWaVUjsayNFeAiydTxUwViQPxN0Hf8dp4FmCQi0UUFovFsEyRSUmFH3MA==} + micromark-util-combine-extensions@1.1.0: + resolution: {integrity: sha512-Q20sp4mfNf9yEqDL50WwuWZHUrCO4fEyeDCnMGmG5Pr0Cz15Uo7KBs6jq+dq0EgX4DPwwrh9m0X+zPV1ypFvUA==} - micromark-util-decode-numeric-character-reference@1.0.0: - resolution: {integrity: sha512-OzO9AI5VUtrTD7KSdagf4MWgHMtET17Ua1fIpXTpuhclCqD8egFWo85GxSGvxgkGS74bEahvtM0WP0HjvV0e4w==} + micromark-util-decode-numeric-character-reference@1.1.0: + resolution: {integrity: sha512-m9V0ExGv0jB1OT21mrWcuf4QhP46pH1KkfWy9ZEezqHKAxkj4mPCy3nIH1rkbdMlChLHX531eOrymlwyZIf2iw==} - micromark-util-decode-string@1.0.2: - resolution: {integrity: sha512-DLT5Ho02qr6QWVNYbRZ3RYOSSWWFuH3tJexd3dgN1odEuPNxCngTCXJum7+ViRAd9BbdxCvMToPOD/IvVhzG6Q==} + micromark-util-decode-string@1.1.0: + resolution: {integrity: sha512-YphLGCK8gM1tG1bd54azwyrQRjCFcmgj2S2GoJDNnh4vYtnL38JS8M4gpxzOPNyHdNEpheyWXCTnnTDY3N+NVQ==} - micromark-util-encode@1.0.1: - resolution: {integrity: sha512-U2s5YdnAYexjKDel31SVMPbfi+eF8y1U4pfiRW/Y8EFVCy/vgxk/2wWTxzcqE71LHtCuCzlBDRU2a5CQ5j+mQA==} + micromark-util-encode@1.1.0: + resolution: {integrity: sha512-EuEzTWSTAj9PA5GOAs992GzNh2dGQO52UvAbtSOMvXTxv3Criqb6IOzJUBCmEqrrXSblJIJBbFFv6zPxpreiJw==} - micromark-util-html-tag-name@1.1.0: - resolution: {integrity: sha512-BKlClMmYROy9UiV03SwNmckkjn8QHVaWkqoAqzivabvdGcwNGMMMH/5szAnywmsTBUzDsU57/mFi0sp4BQO6dA==} + micromark-util-html-tag-name@1.2.0: + resolution: {integrity: sha512-VTQzcuQgFUD7yYztuQFKXT49KghjtETQ+Wv/zUjGSGBioZnkA4P1XXZPT1FHeJA6RwRXSF47yvJ1tsJdoxwO+Q==} - micromark-util-normalize-identifier@1.0.0: - resolution: {integrity: sha512-yg+zrL14bBTFrQ7n35CmByWUTFsgst5JhA4gJYoty4Dqzj4Z4Fr/DHekSS5aLfH9bdlfnSvKAWsAgJhIbogyBg==} + micromark-util-normalize-identifier@1.1.0: + resolution: {integrity: sha512-N+w5vhqrBihhjdpM8+5Xsxy71QWqGn7HYNUvch71iV2PM7+E3uWGox1Qp90loa1ephtCxG2ftRV/Conitc6P2Q==} - micromark-util-resolve-all@1.0.0: - resolution: {integrity: sha512-CB/AGk98u50k42kvgaMM94wzBqozSzDDaonKU7P7jwQIuH2RU0TeBqGYJz2WY1UdihhjweivStrJ2JdkdEmcfw==} + micromark-util-resolve-all@1.1.0: + resolution: {integrity: sha512-b/G6BTMSg+bX+xVCshPTPyAu2tmA0E4X98NSR7eIbeC6ycCqCeE7wjfDIgzEbkzdEVJXRtOG4FbEm/uGbCRouA==} - micromark-util-sanitize-uri@1.1.0: - resolution: {integrity: sha512-RoxtuSCX6sUNtxhbmsEFQfWzs8VN7cTctmBPvYivo98xb/kDEoTCtJQX5wyzIYEmk/lvNFTat4hL8oW0KndFpg==} + micromark-util-sanitize-uri@1.2.0: + resolution: {integrity: sha512-QO4GXv0XZfWey4pYFndLUKEAktKkG5kZTdUNaTAkzbuJxn2tNBOr+QtxR2XpWaMhbImT2dPzyLrPXLlPhph34A==} - micromark-util-subtokenize@1.0.2: - resolution: {integrity: sha512-d90uqCnXp/cy4G881Ub4psE57Sf8YD0pim9QdjCRNjfas2M1u6Lbt+XZK9gnHL2XFhnozZiEdCa9CNfXSfQ6xA==} + micromark-util-subtokenize@1.1.0: + resolution: {integrity: sha512-kUQHyzRoxvZO2PuLzMt2P/dwVsTiivCK8icYTeR+3WgbuPqfHgPPy7nFKbeqRivBvn/3N3GBiNC+JRTMSxEC7A==} - micromark-util-symbol@1.0.1: - resolution: {integrity: sha512-oKDEMK2u5qqAptasDAwWDXq0tG9AssVwAx3E9bBF3t/shRIGsWIRG+cGafs2p/SnDSOecnt6hZPCE2o6lHfFmQ==} + micromark-util-symbol@1.1.0: + resolution: {integrity: sha512-uEjpEYY6KMs1g7QfJ2eX1SQEV+ZT4rUD3UcF6l57acZvLNK7PBZL+ty82Z1qhK1/yXIY4bdx04FKMgR0g4IAag==} - micromark-util-types@1.0.2: - resolution: {integrity: sha512-DCfg/T8fcrhrRKTPjRrw/5LLvdGV7BHySf/1LOZx7TzWZdYRjogNtyNq885z3nNallwr3QUKARjqvHqX1/7t+w==} + micromark-util-types@1.1.0: + resolution: {integrity: sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg==} - micromark@3.1.0: - resolution: {integrity: sha512-6Mj0yHLdUZjHnOPgr5xfWIMqMWS12zDN6iws9SLuSz76W8jTtAv24MN4/CL7gJrl5vtxGInkkqDv/JIoRsQOvA==} + micromark@3.2.0: + resolution: {integrity: sha512-uD66tJj54JLYq0De10AhWycZWGQNUvDI55xPgk2sQM5kn1JYlhbCMTtEeT27+vAhW2FBQxLlOmS3pmA7/2z4aA==} micromatch@4.0.5: resolution: {integrity: sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==} @@ -3562,6 +3779,9 @@ packages: mlly@1.4.0: resolution: {integrity: sha512-ua8PAThnTwpprIaU47EPeZ/bPUVp2QYBbWMphUQpVdBI3Lgqzm5KZQ45Agm3YJedHXaIHl6pBGabaLSUPPSptg==} + mlly@1.7.3: + resolution: {integrity: sha512-xUsx5n/mN0uQf4V548PKQ+YShA4/IW0KI1dZhrNrPCLG+xizETbHTkOa1f8/xut9JRPp8kQuMnz0oqwkTiLo/A==} + mri@1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} @@ -3719,10 +3939,12 @@ packages: npmlog@5.0.1: resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==} + deprecated: This package is no longer supported. npmlog@6.0.2: resolution: {integrity: sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + deprecated: This package is no longer supported. nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} @@ -3758,6 +3980,7 @@ packages: nuxt-simple-sitemap@1.0.11: resolution: {integrity: sha512-FW4kxAKW0FHA3+y2yT790bz4yupHV5qPkf1aaY0Xi3Ct4jKQE4BwX9oKlqz8F4qq3WURETkV7ttzxWmXlNcDsg==} + deprecated: Package has been migrated to @nuxtjs/sitemap. nuxt-unhead@1.4.17: resolution: {integrity: sha512-8DpCE5gmUEKT//KOI+gCzTc315poxNHrbIsOMfv80tyxZIe35VlO14q9wScpL6dE1MxwJ87YtI9rxnq4JZOKLw==} @@ -3866,6 +4089,9 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} + package-manager-detector@0.2.2: + resolution: {integrity: sha512-VgXbyrSNsml4eHWIvxxG/nTL4wgybMTXCV2Un/+yEc3aDKKU6nQBZjbeP3Pl3qm9Qg92X/1ng4ffvCeD/zwHgg==} + pacote@15.2.0: resolution: {integrity: sha512-rJVZeIwHTUta23sIZgEIM62WYwbmGbThdbnkt81ravBplQv+HjyroqnLRNH2+sLJHcGZmLRmhPwACqhfTcOmnA==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -3925,6 +4151,9 @@ packages: path-case@3.0.4: resolution: {integrity: sha512-qO4qCFjXqVTrcbPt/hQfhTQ+VhFsqNKOPtytgNKkKxSoEp3XPUQ8ObFuePylOIok5gjn69ry8XiULxCwot3Wfg==} + path-data-parser@0.1.0: + resolution: {integrity: sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -4001,6 +4230,9 @@ packages: pkg-types@1.0.3: resolution: {integrity: sha512-nN7pYi0AQqJnoLPC9eHFQ8AcyaixBUOwvqc5TDnIKCMEE6I0y8P7OKA7fPexsXGCGxQDl/cmrLAp26LhcwxZ4A==} + pkg-types@1.2.1: + resolution: {integrity: sha512-sQoqa8alT3nHjGuTjuKgOnvjo4cljkufdtLMnO2LBP/wRwuDlo1tkaEdMxCRhyGRPacv/ztlZgDPm2b7FAmEvw==} + playwright-core@1.31.2: resolution: {integrity: sha512-a1dFgCNQw4vCsG7bnojZjDnPewZcw7tZUNFN0ZkcLYKj+mPmXvg4MpaaKZ5SgqPsOmqIf2YsVRkgqiRDxD+fDQ==} engines: {node: '>=14'} @@ -4010,6 +4242,12 @@ packages: resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} engines: {node: '>=4'} + points-on-curve@0.2.0: + resolution: {integrity: sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==} + + points-on-path@0.2.1: + resolution: {integrity: sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g==} + postcss-calc@9.0.1: resolution: {integrity: sha512-TipgjGyzP5QzEhsOZUaIkeO5mKeMFpebWzRogWG/ysonUlnHcq5aJe0jOjpfzUU8PeSaBQnrE8ehR0QA5vs8PQ==} engines: {node: ^14 || ^16 || >=18.0} @@ -4293,6 +4531,7 @@ packages: read-package-json@6.0.3: resolution: {integrity: sha512-4QbpReW4kxFgeBQ0vPAqh2y8sXEB3D4t3jsXbJKIhBiF80KT6XRo45reqwtftju5J6ru1ax06A2Gb/wM1qCOEQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + deprecated: This package is no longer supported. Please use @npmcli/package-json instead. read-pkg-up@7.0.1: resolution: {integrity: sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==} @@ -4414,10 +4653,11 @@ packages: rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true - robust-predicates@3.0.1: - resolution: {integrity: sha512-ndEIpszUHiG4HtDsQLeIuMvRsDnn8c8rYStabochtUeCvfuvNptb5TUbVD68LRAILPX7p9nqQGh4xJgn3EHS/g==} + robust-predicates@3.0.2: + resolution: {integrity: sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==} rollup-plugin-dts@5.2.0: resolution: {integrity: sha512-B68T/haEu2MKcz4kNUhXB8/h5sq4gpplHAJIYNHbh8cp4ZkvzDvNca/11KQdFrB9ZeKucegQIotzo5T0JUtM8w==} @@ -4441,6 +4681,9 @@ packages: engines: {node: '>=14.18.0', npm: '>=8.0.0'} hasBin: true + roughjs@4.6.6: + resolution: {integrity: sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==} + run-applescript@5.0.0: resolution: {integrity: sha512-XcT5rBksx1QdIhlFOCtgZkB99ZEouFZ1E2Kc2LHqNW13U3/74YGdkQRmThTwxy4QIyookibDKYZOPqX//6BlAg==} engines: {node: '>=12'} @@ -4551,6 +4794,7 @@ packages: shiki-es@0.2.0: resolution: {integrity: sha512-RbRMD+IuJJseSZljDdne9ThrUYrwBwJR04FvN4VXpfsU3MNID5VJGHLAD5je/HGThCyEKNgH+nEkSFEWKD7C3Q==} + deprecated: Please migrate to https://github.com/antfu/shikiji signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} @@ -4730,8 +4974,8 @@ packages: peerDependencies: postcss: ^8.2.15 - stylis@4.1.3: - resolution: {integrity: sha512-GP6WDNWf+o403jrEp9c5jibKavrtLW+/qYGhFxFrG8maXhwTBI7gLLhiBb0o7uFccWN+EOS9aMO6cGHWAO07OA==} + stylis@4.3.4: + resolution: {integrity: sha512-osIBl6BGUmSfDkyH2mB7EFvCJntXDrLhKjHTRj/rK6xLH0yuPrHULDRQzKokSOD4VoorhtKpfcfW1GAntu8now==} supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} @@ -4800,6 +5044,9 @@ packages: tinycolor2@1.6.0: resolution: {integrity: sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==} + tinyexec@0.3.1: + resolution: {integrity: sha512-WiCJLEECkO18gwqIp6+hJg0//p23HXp4S+gGtAKu3mI2F2/sXC4FvHvXvB0zJVVaTPhx1/tOwdbRsa1sOBIKqQ==} + tinyws@0.1.0: resolution: {integrity: sha512-6WQ2FlFM7qm6lAXxeKnzsAEfmnBHz5W5EwonNs52V0++YfK1IoCCAWM429afcChFE9BFrDgOFnq7ligaWMsa/A==} engines: {node: '>=12.4'} @@ -4915,6 +5162,9 @@ packages: ufo@1.1.2: resolution: {integrity: sha512-TrY6DsjTQQgyS3E3dBaOXf0TpPD8u9FVrVYmKVegJuFw51n/YB9XPt+U6ydzFG5ZIN7+DIjPbNmXoBj9esYhgQ==} + ufo@1.5.4: + resolution: {integrity: sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ==} + ultrahtml@1.2.0: resolution: {integrity: sha512-vxZM2yNvajRmCj/SknRYGNXk2tqiy6kRNvZjJLaleG3zJbSh/aNkOqD1/CVzypw8tyHyhpzYuwQgMMhUB4ZVNQ==} @@ -5084,8 +5334,8 @@ packages: util@0.12.5: resolution: {integrity: sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==} - uuid@9.0.0: - resolution: {integrity: sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==} + uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} hasBin: true uvu@0.5.6: @@ -5185,6 +5435,10 @@ packages: resolution: {integrity: sha512-wnJA4BnEjOSyFMvjZdpiOwhSq9uDoK8e/kpRJDTaMYzwlkrhG1fwDIZI94CLsLzlCK5cIbMMtFlJlfR57Lavmg==} engines: {node: '>=8.0.0 || >=10.0.0'} + vscode-jsonrpc@8.2.0: + resolution: {integrity: sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==} + engines: {node: '>=14.0.0'} + vscode-languageclient@7.0.0: resolution: {integrity: sha512-P9AXdAPlsCgslpP9pRxYPqkNYV7Xq8300/aZDpO35j1fJm/ncize8iGswzYlcvFw5DQUx4eVk+KvfXdL0rehNg==} engines: {vscode: ^1.52.0} @@ -5192,19 +5446,35 @@ packages: vscode-languageserver-protocol@3.16.0: resolution: {integrity: sha512-sdeUoAawceQdgIfTI+sdcwkiK2KU+2cbEYA0agzM2uqaUy2UpnnGHtWTHVEtS0ES4zHU0eMFRGN+oQgDxlD66A==} + vscode-languageserver-protocol@3.17.5: + resolution: {integrity: sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==} + + vscode-languageserver-textdocument@1.0.12: + resolution: {integrity: sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA==} + vscode-languageserver-textdocument@1.0.8: resolution: {integrity: sha512-1bonkGqQs5/fxGT5UchTgjGVnfysL0O8v1AYMBjqTbWQTFn721zaPGDYFkOKtfDgFiSgXM3KwaG3FMGfW4Ed9Q==} vscode-languageserver-types@3.16.0: resolution: {integrity: sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA==} + vscode-languageserver-types@3.17.5: + resolution: {integrity: sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==} + vscode-languageserver@7.0.0: resolution: {integrity: sha512-60HTx5ID+fLRcgdHfmz0LDZAXYEV68fzwG0JWwEPBode9NuMYTIxuYXPg4ngO8i8+Ou0lM7y6GzaYWbiDL0drw==} hasBin: true + vscode-languageserver@9.0.1: + resolution: {integrity: sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==} + hasBin: true + vscode-uri@3.0.7: resolution: {integrity: sha512-eOpPHogvorZRobNqJGhapa0JdwaxpjVvyBp0QIUMRMSf8ZAlqOdEquKuRmw9Qwu0qXtJIWqFtMkmvJjUZmMjVA==} + vscode-uri@3.0.8: + resolution: {integrity: sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==} + vue-bundle-renderer@1.0.3: resolution: {integrity: sha512-EfjX+5TTUl70bki9hPuVp+54JiZOvFIfoWBcfXsSwLzKEiDYyHNi5iX8srnqLIv3YRnvxgbntdcG1WPq0MvffQ==} @@ -5259,8 +5529,8 @@ packages: resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} engines: {node: '>= 8'} - web-worker@1.2.0: - resolution: {integrity: sha512-PgF341avzqyx60neE9DD+XS26MMNMoUQRz9NOZwW32nPQrF6p77f1htcnjBSEV8BGMKZ16choqUG4hyI0Hx7mA==} + web-worker@1.3.0: + resolution: {integrity: sha512-BSR9wyRsy/KOValMgd5kMyr3JzpdeoR9KVId8u5GVlTTAtNChlsE4yTxeY7zMdNSyOmoKBv8NH2qeRY9Tg+IaA==} webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -5405,12 +5675,12 @@ snapshots: '@jridgewell/gen-mapping': 0.1.1 '@jridgewell/trace-mapping': 0.3.17 - '@aneoconsultingfr/armonik-docs-theme@0.6.12(@types/node@18.15.5)(@unhead/vue@1.1.28(vue@3.3.4))(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(postcss@8.4.24)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6))(vue@3.3.4)': + '@aneoconsultingfr/armonik-docs-theme@0.6.12(@types/node@18.15.5)(@unhead/vue@1.1.28(vue@3.3.4))(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(postcss@8.4.24)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6))(vue@3.3.4)': dependencies: - '@nuxt-themes/docus': 1.12.3(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4))(postcss@8.4.24)(rollup@3.25.3)(vue@3.3.4) - '@nuxt/devtools': 0.6.4(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4))(rollup@3.25.3)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6)) - mermaid: 10.2.3 - nuxt: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4) + '@nuxt-themes/docus': 1.12.3(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5))(postcss@8.4.24)(rollup@3.25.3)(vue@3.3.4) + '@nuxt/devtools': 0.6.4(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5))(rollup@3.25.3)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6)) + mermaid: 10.9.3 + nuxt: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5) nuxt-seo-kit: 1.3.9(@unhead/vue@1.1.28(vue@3.3.4))(rollup@3.25.3)(vue@3.3.4) transitivePeerDependencies: - '@azure/app-configuration' @@ -5450,6 +5720,13 @@ snapshots: - vue - vue-tsc + '@antfu/install-pkg@0.4.1': + dependencies: + package-manager-detector: 0.2.2 + tinyexec: 0.3.1 + + '@antfu/utils@0.7.10': {} + '@antfu/utils@0.7.4': {} '@babel/code-frame@7.21.4': @@ -5646,7 +5923,7 @@ snapshots: '@babel/helper-split-export-declaration': 7.18.6 '@babel/parser': 7.22.5 '@babel/types': 7.22.5 - debug: 4.3.4 + debug: 4.3.7 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -5663,7 +5940,26 @@ snapshots: '@babel/helper-validator-identifier': 7.22.5 to-fast-properties: 2.0.0 - '@braintree/sanitize-url@6.0.2': {} + '@braintree/sanitize-url@6.0.4': {} + + '@braintree/sanitize-url@7.1.0': {} + + '@chevrotain/cst-dts-gen@11.0.3': + dependencies: + '@chevrotain/gast': 11.0.3 + '@chevrotain/types': 11.0.3 + lodash-es: 4.17.21 + + '@chevrotain/gast@11.0.3': + dependencies: + '@chevrotain/types': 11.0.3 + lodash-es: 4.17.21 + + '@chevrotain/regexp-to-ast@11.0.3': {} + + '@chevrotain/types@11.0.3': {} + + '@chevrotain/utils@11.0.3': {} '@cloudflare/kv-asset-handler@0.3.0': dependencies: @@ -5843,7 +6139,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.3.4 + debug: 4.3.7 espree: 9.6.1 globals: 13.20.0 ignore: 5.2.4 @@ -5892,6 +6188,18 @@ snapshots: '@iconify/types@2.0.0': {} + '@iconify/utils@2.1.33': + dependencies: + '@antfu/install-pkg': 0.4.1 + '@antfu/utils': 0.7.10 + '@iconify/types': 2.0.0 + debug: 4.3.7 + kolorist: 1.8.0 + local-pkg: 0.5.0 + mlly: 1.7.3 + transitivePeerDependencies: + - supports-color + '@iconify/vue@4.1.0(vue@3.3.4)': dependencies: '@iconify/types': 2.0.0 @@ -5950,6 +6258,10 @@ snapshots: - encoding - supports-color + '@mermaid-js/parser@0.3.0': + dependencies: + langium: 3.0.0 + '@netlify/functions@1.6.0': dependencies: is-promise: 4.0.0 @@ -6015,14 +6327,14 @@ snapshots: - bluebird - supports-color - '@nuxt-themes/docus@1.12.3(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4))(postcss@8.4.24)(rollup@3.25.3)(vue@3.3.4)': + '@nuxt-themes/docus@1.12.3(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5))(postcss@8.4.24)(rollup@3.25.3)(vue@3.3.4)': dependencies: '@nuxt-themes/elements': 0.9.4(postcss@8.4.24)(rollup@3.25.3)(vue@3.3.4) '@nuxt-themes/tokens': 1.9.1(postcss@8.4.24)(rollup@3.25.3)(vue@3.3.4) '@nuxt-themes/typography': 0.11.0(postcss@8.4.24)(rollup@3.25.3)(vue@3.3.4) '@nuxt/content': 2.6.0(rollup@3.25.3) '@nuxthq/studio': 0.13.2(rollup@3.25.3) - '@vueuse/nuxt': 10.1.2(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4))(rollup@3.25.3)(vue@3.3.4) + '@vueuse/nuxt': 10.1.2(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5))(rollup@3.25.3)(vue@3.3.4) transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -6136,12 +6448,12 @@ snapshots: '@nuxt/devalue@2.0.2': {} - '@nuxt/devtools-kit@0.6.4(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4))(rollup@3.25.3)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6))': + '@nuxt/devtools-kit@0.6.4(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5))(rollup@3.25.3)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6))': dependencies: '@nuxt/kit': 3.6.1(rollup@3.25.3) '@nuxt/schema': 3.6.1(rollup@3.25.3) execa: 7.1.1 - nuxt: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4) + nuxt: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5) vite: 4.3.9(@types/node@18.15.5)(terser@5.17.6) transitivePeerDependencies: - rollup @@ -6161,9 +6473,9 @@ snapshots: rc9: 2.1.1 semver: 7.5.3 - '@nuxt/devtools@0.6.4(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4))(rollup@3.25.3)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6))': + '@nuxt/devtools@0.6.4(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5))(rollup@3.25.3)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6))': dependencies: - '@nuxt/devtools-kit': 0.6.4(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4))(rollup@3.25.3)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6)) + '@nuxt/devtools-kit': 0.6.4(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5))(rollup@3.25.3)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6)) '@nuxt/devtools-wizard': 0.6.4 '@nuxt/kit': 3.6.1(rollup@3.25.3) birpc: 0.2.12 @@ -6181,7 +6493,7 @@ snapshots: launch-editor: 2.6.0 local-pkg: 0.4.3 magicast: 0.2.9 - nuxt: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4) + nuxt: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5) nypm: 0.2.1 pacote: 15.2.0 pathe: 1.1.1 @@ -6206,18 +6518,18 @@ snapshots: - supports-color - utf-8-validate - '@nuxt/eslint-config@0.3.8(eslint@9.0.0)(typescript@5.0.4)': + '@nuxt/eslint-config@0.3.8(eslint@9.0.0)(typescript@4.9.5)': dependencies: '@eslint/js': 9.0.0 - '@nuxt/eslint-plugin': 0.3.8(eslint@9.0.0)(typescript@5.0.4) + '@nuxt/eslint-plugin': 0.3.8(eslint@9.0.0)(typescript@4.9.5) '@rushstack/eslint-patch': 1.10.2 - '@stylistic/eslint-plugin': 1.7.2(eslint@9.0.0)(typescript@5.0.4) - '@typescript-eslint/eslint-plugin': 7.7.0(@typescript-eslint/parser@7.7.0(eslint@9.0.0)(typescript@5.0.4))(eslint@9.0.0)(typescript@5.0.4) - '@typescript-eslint/parser': 7.7.0(eslint@9.0.0)(typescript@5.0.4) + '@stylistic/eslint-plugin': 1.7.2(eslint@9.0.0)(typescript@4.9.5) + '@typescript-eslint/eslint-plugin': 7.7.0(@typescript-eslint/parser@7.7.0(eslint@9.0.0)(typescript@4.9.5))(eslint@9.0.0)(typescript@4.9.5) + '@typescript-eslint/parser': 7.7.0(eslint@9.0.0)(typescript@4.9.5) eslint: 9.0.0 eslint-config-flat-gitignore: 0.1.5 eslint-flat-config-utils: 0.2.3 - eslint-plugin-import-x: 0.5.0(eslint@9.0.0)(typescript@5.0.4) + eslint-plugin-import-x: 0.5.0(eslint@9.0.0)(typescript@4.9.5) eslint-plugin-jsdoc: 48.2.3(eslint@9.0.0) eslint-plugin-unicorn: 52.0.0(eslint@9.0.0) eslint-plugin-vue: 9.25.0(eslint@9.0.0) @@ -6229,10 +6541,10 @@ snapshots: - supports-color - typescript - '@nuxt/eslint-plugin@0.3.8(eslint@9.0.0)(typescript@5.0.4)': + '@nuxt/eslint-plugin@0.3.8(eslint@9.0.0)(typescript@4.9.5)': dependencies: '@typescript-eslint/types': 7.7.0 - '@typescript-eslint/utils': 7.7.0(eslint@9.0.0)(typescript@5.0.4) + '@typescript-eslint/utils': 7.7.0(eslint@9.0.0)(typescript@4.9.5) eslint: 9.0.0 transitivePeerDependencies: - supports-color @@ -6339,7 +6651,7 @@ snapshots: defu: 6.1.2 hookable: 5.5.3 jiti: 1.18.2 - pathe: 1.1.1 + pathe: 1.1.2 pkg-types: 1.0.3 postcss-import-resolver: 2.0.0 scule: 1.0.0 @@ -6355,7 +6667,7 @@ snapshots: dependencies: defu: 6.1.2 hookable: 5.5.3 - pathe: 1.1.1 + pathe: 1.1.2 pkg-types: 1.0.3 postcss-import-resolver: 2.0.0 std-env: 3.3.3 @@ -6424,7 +6736,7 @@ snapshots: '@nuxt/ui-templates@1.2.0': {} - '@nuxt/vite-builder@3.6.1(@types/node@18.15.5)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4)(vue@3.3.4)': + '@nuxt/vite-builder@3.6.1(@types/node@18.15.5)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5)(vue@3.3.4)': dependencies: '@nuxt/kit': 3.6.1(rollup@3.25.3) '@rollup/plugin-replace': 5.0.2(rollup@3.25.3) @@ -6459,7 +6771,7 @@ snapshots: unplugin: 1.3.1 vite: 4.3.9(@types/node@18.15.5)(terser@5.17.6) vite-node: 0.32.2(@types/node@18.15.5)(terser@5.17.6) - vite-plugin-checker: 0.6.1(eslint@9.0.0)(optionator@0.9.3)(typescript@5.0.4)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6)) + vite-plugin-checker: 0.6.1(eslint@9.0.0)(optionator@0.9.3)(typescript@4.9.5)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6)) vue: 3.3.4 vue-bundle-renderer: 1.0.3 transitivePeerDependencies: @@ -6635,31 +6947,31 @@ snapshots: estraverse: 5.3.0 picomatch: 4.0.2 - '@stylistic/eslint-plugin-plus@1.7.2(eslint@9.0.0)(typescript@5.0.4)': + '@stylistic/eslint-plugin-plus@1.7.2(eslint@9.0.0)(typescript@4.9.5)': dependencies: '@types/eslint': 8.56.9 - '@typescript-eslint/utils': 6.21.0(eslint@9.0.0)(typescript@5.0.4) + '@typescript-eslint/utils': 6.21.0(eslint@9.0.0)(typescript@4.9.5) eslint: 9.0.0 transitivePeerDependencies: - supports-color - typescript - '@stylistic/eslint-plugin-ts@1.7.2(eslint@9.0.0)(typescript@5.0.4)': + '@stylistic/eslint-plugin-ts@1.7.2(eslint@9.0.0)(typescript@4.9.5)': dependencies: '@stylistic/eslint-plugin-js': 1.7.2(eslint@9.0.0) '@types/eslint': 8.56.9 - '@typescript-eslint/utils': 6.21.0(eslint@9.0.0)(typescript@5.0.4) + '@typescript-eslint/utils': 6.21.0(eslint@9.0.0)(typescript@4.9.5) eslint: 9.0.0 transitivePeerDependencies: - supports-color - typescript - '@stylistic/eslint-plugin@1.7.2(eslint@9.0.0)(typescript@5.0.4)': + '@stylistic/eslint-plugin@1.7.2(eslint@9.0.0)(typescript@4.9.5)': dependencies: '@stylistic/eslint-plugin-js': 1.7.2(eslint@9.0.0) '@stylistic/eslint-plugin-jsx': 1.7.2(eslint@9.0.0) - '@stylistic/eslint-plugin-plus': 1.7.2(eslint@9.0.0)(typescript@5.0.4) - '@stylistic/eslint-plugin-ts': 1.7.2(eslint@9.0.0)(typescript@5.0.4) + '@stylistic/eslint-plugin-plus': 1.7.2(eslint@9.0.0)(typescript@4.9.5) + '@stylistic/eslint-plugin-ts': 1.7.2(eslint@9.0.0)(typescript@4.9.5) '@types/eslint': 8.56.9 eslint: 9.0.0 transitivePeerDependencies: @@ -6677,9 +6989,130 @@ snapshots: '@tufjs/canonical-json': 1.0.0 minimatch: 9.0.1 - '@types/debug@4.1.7': + '@types/d3-array@3.2.1': {} + + '@types/d3-axis@3.0.6': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-brush@3.0.6': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-chord@3.0.6': {} + + '@types/d3-color@3.1.3': {} + + '@types/d3-contour@3.0.6': + dependencies: + '@types/d3-array': 3.2.1 + '@types/geojson': 7946.0.14 + + '@types/d3-delaunay@6.0.4': {} + + '@types/d3-dispatch@3.0.6': {} + + '@types/d3-drag@3.0.7': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-dsv@3.0.7': {} + + '@types/d3-ease@3.0.2': {} + + '@types/d3-fetch@3.0.7': + dependencies: + '@types/d3-dsv': 3.0.7 + + '@types/d3-force@3.0.10': {} + + '@types/d3-format@3.0.4': {} + + '@types/d3-geo@3.1.0': + dependencies: + '@types/geojson': 7946.0.14 + + '@types/d3-hierarchy@3.1.7': {} + + '@types/d3-interpolate@3.0.4': + dependencies: + '@types/d3-color': 3.1.3 + + '@types/d3-path@3.1.0': {} + + '@types/d3-polygon@3.0.2': {} + + '@types/d3-quadtree@3.0.6': {} + + '@types/d3-random@3.0.3': {} + + '@types/d3-scale-chromatic@3.0.3': {} + + '@types/d3-scale@4.0.8': dependencies: - '@types/ms': 0.7.31 + '@types/d3-time': 3.0.3 + + '@types/d3-selection@3.0.11': {} + + '@types/d3-shape@3.1.6': + dependencies: + '@types/d3-path': 3.1.0 + + '@types/d3-time-format@4.0.3': {} + + '@types/d3-time@3.0.3': {} + + '@types/d3-timer@3.0.2': {} + + '@types/d3-transition@3.0.9': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-zoom@3.0.8': + dependencies: + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + + '@types/d3@7.4.3': + dependencies: + '@types/d3-array': 3.2.1 + '@types/d3-axis': 3.0.6 + '@types/d3-brush': 3.0.6 + '@types/d3-chord': 3.0.6 + '@types/d3-color': 3.1.3 + '@types/d3-contour': 3.0.6 + '@types/d3-delaunay': 6.0.4 + '@types/d3-dispatch': 3.0.6 + '@types/d3-drag': 3.0.7 + '@types/d3-dsv': 3.0.7 + '@types/d3-ease': 3.0.2 + '@types/d3-fetch': 3.0.7 + '@types/d3-force': 3.0.10 + '@types/d3-format': 3.0.4 + '@types/d3-geo': 3.1.0 + '@types/d3-hierarchy': 3.1.7 + '@types/d3-interpolate': 3.0.4 + '@types/d3-path': 3.1.0 + '@types/d3-polygon': 3.0.2 + '@types/d3-quadtree': 3.0.6 + '@types/d3-random': 3.0.3 + '@types/d3-scale': 4.0.8 + '@types/d3-scale-chromatic': 3.0.3 + '@types/d3-selection': 3.0.11 + '@types/d3-shape': 3.1.6 + '@types/d3-time': 3.0.3 + '@types/d3-time-format': 4.0.3 + '@types/d3-timer': 3.0.2 + '@types/d3-transition': 3.0.9 + '@types/d3-zoom': 3.0.8 + + '@types/debug@4.1.12': + dependencies: + '@types/ms': 0.7.34 + + '@types/dompurify@3.0.5': + dependencies: + '@types/trusted-types': 2.0.7 '@types/eslint@8.56.9': dependencies: @@ -6693,9 +7126,11 @@ snapshots: '@types/jsonfile': 6.1.1 '@types/node': 18.15.5 + '@types/geojson@7946.0.14': {} + '@types/hast@2.3.4': dependencies: - '@types/unist': 2.0.6 + '@types/unist': 3.0.3 '@types/http-proxy@1.17.11': dependencies: @@ -6709,11 +7144,11 @@ snapshots: dependencies: '@types/node': 18.15.5 - '@types/mdast@3.0.10': + '@types/mdast@3.0.15': dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 - '@types/ms@0.7.31': {} + '@types/ms@0.7.34': {} '@types/node@17.0.45': {} @@ -6733,42 +7168,46 @@ snapshots: '@types/semver@7.5.8': {} - '@types/unist@2.0.6': {} + '@types/trusted-types@2.0.7': {} + + '@types/unist@2.0.11': {} + + '@types/unist@3.0.3': {} '@types/web-bluetooth@0.0.16': {} '@types/web-bluetooth@0.0.17': {} - '@typescript-eslint/eslint-plugin@7.7.0(@typescript-eslint/parser@7.7.0(eslint@9.0.0)(typescript@5.0.4))(eslint@9.0.0)(typescript@5.0.4)': + '@typescript-eslint/eslint-plugin@7.7.0(@typescript-eslint/parser@7.7.0(eslint@9.0.0)(typescript@4.9.5))(eslint@9.0.0)(typescript@4.9.5)': dependencies: '@eslint-community/regexpp': 4.10.0 - '@typescript-eslint/parser': 7.7.0(eslint@9.0.0)(typescript@5.0.4) + '@typescript-eslint/parser': 7.7.0(eslint@9.0.0)(typescript@4.9.5) '@typescript-eslint/scope-manager': 7.7.0 - '@typescript-eslint/type-utils': 7.7.0(eslint@9.0.0)(typescript@5.0.4) - '@typescript-eslint/utils': 7.7.0(eslint@9.0.0)(typescript@5.0.4) + '@typescript-eslint/type-utils': 7.7.0(eslint@9.0.0)(typescript@4.9.5) + '@typescript-eslint/utils': 7.7.0(eslint@9.0.0)(typescript@4.9.5) '@typescript-eslint/visitor-keys': 7.7.0 - debug: 4.3.4 + debug: 4.3.7 eslint: 9.0.0 graphemer: 1.4.0 ignore: 5.3.1 natural-compare: 1.4.0 semver: 7.6.0 - ts-api-utils: 1.3.0(typescript@5.0.4) + ts-api-utils: 1.3.0(typescript@4.9.5) optionalDependencies: - typescript: 5.0.4 + typescript: 4.9.5 transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@7.7.0(eslint@9.0.0)(typescript@5.0.4)': + '@typescript-eslint/parser@7.7.0(eslint@9.0.0)(typescript@4.9.5)': dependencies: '@typescript-eslint/scope-manager': 7.7.0 '@typescript-eslint/types': 7.7.0 - '@typescript-eslint/typescript-estree': 7.7.0(typescript@5.0.4) + '@typescript-eslint/typescript-estree': 7.7.0(typescript@4.9.5) '@typescript-eslint/visitor-keys': 7.7.0 - debug: 4.3.4 + debug: 4.3.7 eslint: 9.0.0 optionalDependencies: - typescript: 5.0.4 + typescript: 4.9.5 transitivePeerDependencies: - supports-color @@ -6782,15 +7221,15 @@ snapshots: '@typescript-eslint/types': 7.7.0 '@typescript-eslint/visitor-keys': 7.7.0 - '@typescript-eslint/type-utils@7.7.0(eslint@9.0.0)(typescript@5.0.4)': + '@typescript-eslint/type-utils@7.7.0(eslint@9.0.0)(typescript@4.9.5)': dependencies: - '@typescript-eslint/typescript-estree': 7.7.0(typescript@5.0.4) - '@typescript-eslint/utils': 7.7.0(eslint@9.0.0)(typescript@5.0.4) - debug: 4.3.4 + '@typescript-eslint/typescript-estree': 7.7.0(typescript@4.9.5) + '@typescript-eslint/utils': 7.7.0(eslint@9.0.0)(typescript@4.9.5) + debug: 4.3.7 eslint: 9.0.0 - ts-api-utils: 1.3.0(typescript@5.0.4) + ts-api-utils: 1.3.0(typescript@4.9.5) optionalDependencies: - typescript: 5.0.4 + typescript: 4.9.5 transitivePeerDependencies: - supports-color @@ -6798,58 +7237,58 @@ snapshots: '@typescript-eslint/types@7.7.0': {} - '@typescript-eslint/typescript-estree@6.21.0(typescript@5.0.4)': + '@typescript-eslint/typescript-estree@6.21.0(typescript@4.9.5)': dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.4 + debug: 4.3.7 globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 semver: 7.6.0 - ts-api-utils: 1.3.0(typescript@5.0.4) + ts-api-utils: 1.3.0(typescript@4.9.5) optionalDependencies: - typescript: 5.0.4 + typescript: 4.9.5 transitivePeerDependencies: - supports-color - '@typescript-eslint/typescript-estree@7.7.0(typescript@5.0.4)': + '@typescript-eslint/typescript-estree@7.7.0(typescript@4.9.5)': dependencies: '@typescript-eslint/types': 7.7.0 '@typescript-eslint/visitor-keys': 7.7.0 - debug: 4.3.4 + debug: 4.3.7 globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.4 semver: 7.6.0 - ts-api-utils: 1.3.0(typescript@5.0.4) + ts-api-utils: 1.3.0(typescript@4.9.5) optionalDependencies: - typescript: 5.0.4 + typescript: 4.9.5 transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@6.21.0(eslint@9.0.0)(typescript@5.0.4)': + '@typescript-eslint/utils@6.21.0(eslint@9.0.0)(typescript@4.9.5)': dependencies: '@eslint-community/eslint-utils': 4.4.0(eslint@9.0.0) '@types/json-schema': 7.0.15 '@types/semver': 7.5.0 '@typescript-eslint/scope-manager': 6.21.0 '@typescript-eslint/types': 6.21.0 - '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.0.4) + '@typescript-eslint/typescript-estree': 6.21.0(typescript@4.9.5) eslint: 9.0.0 semver: 7.6.0 transitivePeerDependencies: - supports-color - typescript - '@typescript-eslint/utils@7.7.0(eslint@9.0.0)(typescript@5.0.4)': + '@typescript-eslint/utils@7.7.0(eslint@9.0.0)(typescript@4.9.5)': dependencies: '@eslint-community/eslint-utils': 4.4.0(eslint@9.0.0) '@types/json-schema': 7.0.15 '@types/semver': 7.5.8 '@typescript-eslint/scope-manager': 7.7.0 '@typescript-eslint/types': 7.7.0 - '@typescript-eslint/typescript-estree': 7.7.0(typescript@5.0.4) + '@typescript-eslint/typescript-estree': 7.7.0(typescript@4.9.5) eslint: 9.0.0 semver: 7.6.0 transitivePeerDependencies: @@ -7103,13 +7542,13 @@ snapshots: '@vueuse/metadata@9.13.0': {} - '@vueuse/nuxt@10.1.2(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4))(rollup@3.25.3)(vue@3.3.4)': + '@vueuse/nuxt@10.1.2(nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5))(rollup@3.25.3)(vue@3.3.4)': dependencies: '@nuxt/kit': 3.6.1(rollup@3.25.3) '@vueuse/core': 10.1.2(vue@3.3.4) '@vueuse/metadata': 10.1.2 local-pkg: 0.4.3 - nuxt: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4) + nuxt: 3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5) vue-demi: 0.14.0(vue@3.3.4) transitivePeerDependencies: - '@vue/composition-api' @@ -7143,23 +7582,25 @@ snapshots: acorn@8.11.3: {} + acorn@8.14.0: {} + acorn@8.9.0: {} agent-base@6.0.2: dependencies: - debug: 4.3.4 + debug: 4.3.7 transitivePeerDependencies: - supports-color agent-base@7.1.0: dependencies: - debug: 4.3.4 + debug: 4.3.7 transitivePeerDependencies: - supports-color agentkeepalive@4.3.0: dependencies: - debug: 4.3.4 + debug: 4.3.7 depd: 2.0.0 humanize-ms: 1.2.1 transitivePeerDependencies: @@ -7553,6 +7994,20 @@ snapshots: parse5: 7.1.2 parse5-htmlparser2-tree-adapter: 7.0.0 + chevrotain-allstar@0.3.1(chevrotain@11.0.3): + dependencies: + chevrotain: 11.0.3 + lodash-es: 4.17.21 + + chevrotain@11.0.3: + dependencies: + '@chevrotain/cst-dts-gen': 11.0.3 + '@chevrotain/gast': 11.0.3 + '@chevrotain/regexp-to-ast': 11.0.3 + '@chevrotain/types': 11.0.3 + '@chevrotain/utils': 11.0.3 + lodash-es: 4.17.21 + chokidar@3.5.3: dependencies: anymatch: 3.1.3 @@ -7665,6 +8120,8 @@ snapshots: concat-map@0.0.1: {} + confbox@0.1.8: {} + consola@2.15.3: {} consola@3.1.0: {} @@ -7799,22 +8256,23 @@ snapshots: cuint@0.2.2: {} - cytoscape-cose-bilkent@4.1.0(cytoscape@3.23.0): + cytoscape-cose-bilkent@4.1.0(cytoscape@3.30.3): dependencies: cose-base: 1.0.3 - cytoscape: 3.23.0 + cytoscape: 3.30.3 - cytoscape-fcose@2.2.0(cytoscape@3.23.0): + cytoscape-fcose@2.2.0(cytoscape@3.30.3): dependencies: cose-base: 2.2.0 - cytoscape: 3.23.0 + cytoscape: 3.30.3 - cytoscape@3.23.0: + cytoscape@3.30.3: {} + + d3-array@2.12.1: dependencies: - heap: 0.2.7 - lodash: 4.17.21 + internmap: 1.0.1 - d3-array@3.2.2: + d3-array@3.2.4: dependencies: internmap: 2.0.3 @@ -7836,11 +8294,11 @@ snapshots: d3-contour@4.0.2: dependencies: - d3-array: 3.2.2 + d3-array: 3.2.4 - d3-delaunay@6.0.2: + d3-delaunay@6.0.4: dependencies: - delaunator: 5.0.0 + delaunator: 5.0.1 d3-dispatch@3.0.1: {} @@ -7869,9 +8327,9 @@ snapshots: d3-format@3.1.0: {} - d3-geo@3.1.0: + d3-geo@3.1.1: dependencies: - d3-array: 3.2.2 + d3-array: 3.2.4 d3-hierarchy@3.1.2: {} @@ -7879,6 +8337,8 @@ snapshots: dependencies: d3-color: 3.1.0 + d3-path@1.0.9: {} + d3-path@3.1.0: {} d3-polygon@3.0.1: {} @@ -7887,14 +8347,19 @@ snapshots: d3-random@3.0.1: {} - d3-scale-chromatic@3.0.0: + d3-sankey@0.12.3: + dependencies: + d3-array: 2.12.1 + d3-shape: 1.3.7 + + d3-scale-chromatic@3.1.0: dependencies: d3-color: 3.1.0 d3-interpolate: 3.0.1 d3-scale@4.0.2: dependencies: - d3-array: 3.2.2 + d3-array: 3.2.4 d3-format: 3.1.0 d3-interpolate: 3.0.1 d3-time: 3.1.0 @@ -7902,6 +8367,10 @@ snapshots: d3-selection@3.0.0: {} + d3-shape@1.3.7: + dependencies: + d3-path: 1.0.9 + d3-shape@3.2.0: dependencies: d3-path: 3.1.0 @@ -7912,7 +8381,7 @@ snapshots: d3-time@3.1.0: dependencies: - d3-array: 3.2.2 + d3-array: 3.2.4 d3-timer@3.0.1: {} @@ -7933,15 +8402,15 @@ snapshots: d3-selection: 3.0.0 d3-transition: 3.0.1(d3-selection@3.0.0) - d3@7.8.2: + d3@7.9.0: dependencies: - d3-array: 3.2.2 + d3-array: 3.2.4 d3-axis: 3.0.0 d3-brush: 3.0.0 d3-chord: 3.0.1 d3-color: 3.1.0 d3-contour: 4.0.2 - d3-delaunay: 6.0.2 + d3-delaunay: 6.0.4 d3-dispatch: 3.0.1 d3-drag: 3.0.0 d3-dsv: 3.0.1 @@ -7949,7 +8418,7 @@ snapshots: d3-fetch: 3.0.1 d3-force: 3.0.0 d3-format: 3.1.0 - d3-geo: 3.1.0 + d3-geo: 3.1.1 d3-hierarchy: 3.1.2 d3-interpolate: 3.0.1 d3-path: 3.1.0 @@ -7957,7 +8426,7 @@ snapshots: d3-quadtree: 3.0.1 d3-random: 3.0.1 d3-scale: 4.0.2 - d3-scale-chromatic: 3.0.0 + d3-scale-chromatic: 3.1.0 d3-selection: 3.0.0 d3-shape: 3.2.0 d3-time: 3.1.0 @@ -7968,12 +8437,17 @@ snapshots: dagre-d3-es@7.0.10: dependencies: - d3: 7.8.2 + d3: 7.9.0 + lodash-es: 4.17.21 + + dagre-d3-es@7.0.11: + dependencies: + d3: 7.9.0 lodash-es: 4.17.21 data-uri-to-buffer@4.0.1: {} - dayjs@1.11.7: {} + dayjs@1.11.13: {} de-indent@1.0.2: {} @@ -7989,6 +8463,10 @@ snapshots: dependencies: ms: 2.1.2 + debug@4.3.7: + dependencies: + ms: 2.1.3 + decode-named-character-reference@1.0.2: dependencies: character-entities: 2.0.2 @@ -8064,9 +8542,9 @@ snapshots: defu@6.1.2: {} - delaunator@5.0.0: + delaunator@5.0.1: dependencies: - robust-predicates: 3.0.1 + robust-predicates: 3.0.2 delayed-stream@1.0.0: {} @@ -8092,6 +8570,8 @@ snapshots: diff@5.1.0: {} + diff@5.2.0: {} + dir-glob@3.0.1: dependencies: path-type: 4.0.0 @@ -8112,7 +8592,7 @@ snapshots: dependencies: domelementtype: 2.3.0 - dompurify@3.0.3: {} + dompurify@3.1.6: {} domutils@3.0.1: dependencies: @@ -8143,7 +8623,7 @@ snapshots: electron-to-chromium@1.4.740: {} - elkjs@0.8.2: {} + elkjs@0.9.3: {} emoji-regex@10.2.1: {} @@ -8167,7 +8647,7 @@ snapshots: engine.io-client@6.4.0: dependencies: '@socket.io/component-emitter': 3.1.0 - debug: 4.3.4 + debug: 4.3.7 engine.io-parser: 5.0.6 ws: 8.11.0 xmlhttprequest-ssl: 2.0.0 @@ -8285,10 +8765,10 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-plugin-import-x@0.5.0(eslint@9.0.0)(typescript@5.0.4): + eslint-plugin-import-x@0.5.0(eslint@9.0.0)(typescript@4.9.5): dependencies: - '@typescript-eslint/utils': 7.7.0(eslint@9.0.0)(typescript@5.0.4) - debug: 4.3.4 + '@typescript-eslint/utils': 7.7.0(eslint@9.0.0)(typescript@4.9.5) + debug: 4.3.7 doctrine: 3.0.0 eslint: 9.0.0 eslint-import-resolver-node: 0.3.9 @@ -8305,7 +8785,7 @@ snapshots: '@es-joy/jsdoccomment': 0.42.0 are-docs-informative: 0.0.2 comment-parser: 1.4.1 - debug: 4.3.4 + debug: 4.3.7 escape-string-regexp: 4.0.0 eslint: 9.0.0 esquery: 1.5.0 @@ -8802,6 +9282,8 @@ snapshots: ufo: 1.1.2 uncrypto: 0.1.3 + hachure-fill@0.5.2: {} + has-flag@3.0.0: {} has-flag@4.0.0: {} @@ -8831,7 +9313,7 @@ snapshots: hast-util-from-parse5@7.1.2: dependencies: '@types/hast': 2.3.4 - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 hastscript: 7.2.0 property-information: 6.2.0 vfile: 5.3.7 @@ -8847,7 +9329,7 @@ snapshots: hast-util-is-element@2.1.3: dependencies: '@types/hast': 2.3.4 - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 hast-util-parse-selector@3.1.1: dependencies: @@ -8895,8 +9377,6 @@ snapshots: capital-case: 1.0.4 tslib: 2.5.0 - heap@0.2.7: {} - hookable@5.5.3: {} hosted-git-info@2.8.9: {} @@ -8928,7 +9408,7 @@ snapshots: http-graceful-shutdown@3.1.13: dependencies: - debug: 4.3.4 + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -8936,7 +9416,7 @@ snapshots: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.3.4 + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -8953,14 +9433,14 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.4 + debug: 4.3.7 transitivePeerDependencies: - supports-color https-proxy-agent@7.0.0: dependencies: agent-base: 7.1.0 - debug: 4.3.4 + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -9034,13 +9514,15 @@ snapshots: through: 2.3.8 wrap-ansi: 6.2.0 + internmap@1.0.1: {} + internmap@2.0.3: {} ioredis@5.3.2: dependencies: '@ioredis/commands': 1.2.0 cluster-key-slot: 1.1.2 - debug: 4.3.4 + debug: 4.3.7 denque: 2.1.0 lodash.defaults: 4.2.0 lodash.isarguments: 3.1.0 @@ -9240,11 +9722,15 @@ snapshots: jsonparse@1.3.1: {} + katex@0.16.11: + dependencies: + commander: 8.3.0 + keyv@4.5.4: dependencies: json-buffer: 3.0.1 - khroma@2.0.0: {} + khroma@2.1.0: {} kleur@3.0.3: {} @@ -9256,6 +9742,14 @@ snapshots: kolorist@1.8.0: {} + langium@3.0.0: + dependencies: + chevrotain: 11.0.3 + chevrotain-allstar: 0.3.1(chevrotain@11.0.3) + vscode-languageserver: 9.0.1 + vscode-languageserver-textdocument: 1.0.12 + vscode-uri: 3.0.8 + launch-editor@2.6.0: dependencies: picocolors: 1.0.0 @@ -9307,6 +9801,11 @@ snapshots: local-pkg@0.4.3: {} + local-pkg@0.5.0: + dependencies: + mlly: 1.7.3 + pkg-types: 1.0.3 + locate-path@5.0.0: dependencies: p-locate: 4.1.0 @@ -9482,38 +9981,40 @@ snapshots: markdown-it: 13.0.1 markdownlint-micromark: 0.1.5 + marked@13.0.3: {} + marky@1.2.5: {} mdast-squeeze-paragraphs@5.2.1: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 unist-util-visit: 4.1.2 mdast-util-definitions@5.1.2: dependencies: - '@types/mdast': 3.0.10 - '@types/unist': 2.0.6 + '@types/mdast': 3.0.15 + '@types/unist': 2.0.11 unist-util-visit: 4.1.2 mdast-util-find-and-replace@2.2.2: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 escape-string-regexp: 5.0.0 unist-util-is: 5.2.0 unist-util-visit-parents: 5.1.3 - mdast-util-from-markdown@1.3.0: + mdast-util-from-markdown@1.3.1: dependencies: - '@types/mdast': 3.0.10 - '@types/unist': 2.0.6 + '@types/mdast': 3.0.15 + '@types/unist': 2.0.11 decode-named-character-reference: 1.0.2 - mdast-util-to-string: 3.1.1 - micromark: 3.1.0 - micromark-util-decode-numeric-character-reference: 1.0.0 - micromark-util-decode-string: 1.0.2 - micromark-util-normalize-identifier: 1.0.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + mdast-util-to-string: 3.2.0 + micromark: 3.2.0 + micromark-util-decode-numeric-character-reference: 1.1.0 + micromark-util-decode-string: 1.1.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 unist-util-stringify-position: 3.0.3 uvu: 0.5.6 transitivePeerDependencies: @@ -9521,39 +10022,39 @@ snapshots: mdast-util-gfm-autolink-literal@1.0.3: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 ccount: 2.0.1 mdast-util-find-and-replace: 2.2.2 - micromark-util-character: 1.1.0 + micromark-util-character: 1.2.0 mdast-util-gfm-footnote@1.0.2: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 mdast-util-to-markdown: 1.5.0 - micromark-util-normalize-identifier: 1.0.0 + micromark-util-normalize-identifier: 1.1.0 mdast-util-gfm-strikethrough@1.0.3: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 mdast-util-to-markdown: 1.5.0 mdast-util-gfm-table@1.0.7: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 markdown-table: 3.0.3 - mdast-util-from-markdown: 1.3.0 + mdast-util-from-markdown: 1.3.1 mdast-util-to-markdown: 1.5.0 transitivePeerDependencies: - supports-color mdast-util-gfm-task-list-item@1.0.2: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 mdast-util-to-markdown: 1.5.0 mdast-util-gfm@2.0.2: dependencies: - mdast-util-from-markdown: 1.3.0 + mdast-util-from-markdown: 1.3.1 mdast-util-gfm-autolink-literal: 1.0.3 mdast-util-gfm-footnote: 1.0.2 mdast-util-gfm-strikethrough: 1.0.3 @@ -9565,15 +10066,15 @@ snapshots: mdast-util-phrasing@3.0.1: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 unist-util-is: 5.2.0 mdast-util-to-hast@12.3.0: dependencies: '@types/hast': 2.3.4 - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 mdast-util-definitions: 5.1.2 - micromark-util-sanitize-uri: 1.1.0 + micromark-util-sanitize-uri: 1.2.0 trim-lines: 3.0.1 unist-util-generated: 2.0.1 unist-util-position: 4.0.4 @@ -9581,18 +10082,18 @@ snapshots: mdast-util-to-markdown@1.5.0: dependencies: - '@types/mdast': 3.0.10 - '@types/unist': 2.0.6 + '@types/mdast': 3.0.15 + '@types/unist': 2.0.11 longest-streak: 3.1.0 mdast-util-phrasing: 3.0.1 - mdast-util-to-string: 3.1.1 - micromark-util-decode-string: 1.0.2 + mdast-util-to-string: 3.2.0 + micromark-util-decode-string: 1.1.0 unist-util-visit: 4.1.2 zwitch: 2.0.4 - mdast-util-to-string@3.1.1: + mdast-util-to-string@3.2.0: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 mdn-data@2.0.28: {} @@ -9609,93 +10110,122 @@ snapshots: merge2@1.4.1: {} - mermaid@10.2.3: + mermaid@10.9.3: dependencies: - '@braintree/sanitize-url': 6.0.2 - cytoscape: 3.23.0 - cytoscape-cose-bilkent: 4.1.0(cytoscape@3.23.0) - cytoscape-fcose: 2.2.0(cytoscape@3.23.0) - d3: 7.8.2 + '@braintree/sanitize-url': 6.0.4 + '@types/d3-scale': 4.0.8 + '@types/d3-scale-chromatic': 3.0.3 + cytoscape: 3.30.3 + cytoscape-cose-bilkent: 4.1.0(cytoscape@3.30.3) + d3: 7.9.0 + d3-sankey: 0.12.3 dagre-d3-es: 7.0.10 - dayjs: 1.11.7 - dompurify: 3.0.3 - elkjs: 0.8.2 - khroma: 2.0.0 + dayjs: 1.11.13 + dompurify: 3.1.6 + elkjs: 0.9.3 + katex: 0.16.11 + khroma: 2.1.0 lodash-es: 4.17.21 - mdast-util-from-markdown: 1.3.0 + mdast-util-from-markdown: 1.3.1 non-layered-tidy-tree-layout: 2.0.2 - stylis: 4.1.3 + stylis: 4.3.4 + ts-dedent: 2.2.0 + uuid: 9.0.1 + web-worker: 1.3.0 + transitivePeerDependencies: + - supports-color + + mermaid@11.4.0: + dependencies: + '@braintree/sanitize-url': 7.1.0 + '@iconify/utils': 2.1.33 + '@mermaid-js/parser': 0.3.0 + '@types/d3': 7.4.3 + '@types/dompurify': 3.0.5 + cytoscape: 3.30.3 + cytoscape-cose-bilkent: 4.1.0(cytoscape@3.30.3) + cytoscape-fcose: 2.2.0(cytoscape@3.30.3) + d3: 7.9.0 + d3-sankey: 0.12.3 + dagre-d3-es: 7.0.11 + dayjs: 1.11.13 + dompurify: 3.1.6 + katex: 0.16.11 + khroma: 2.1.0 + lodash-es: 4.17.21 + marked: 13.0.3 + roughjs: 4.6.6 + stylis: 4.3.4 ts-dedent: 2.2.0 - uuid: 9.0.0 - web-worker: 1.2.0 + uuid: 9.0.1 transitivePeerDependencies: - supports-color - micromark-core-commonmark@1.0.6: + micromark-core-commonmark@1.1.0: dependencies: decode-named-character-reference: 1.0.2 - micromark-factory-destination: 1.0.0 - micromark-factory-label: 1.0.2 - micromark-factory-space: 1.0.0 - micromark-factory-title: 1.0.2 - micromark-factory-whitespace: 1.0.0 - micromark-util-character: 1.1.0 - micromark-util-chunked: 1.0.0 - micromark-util-classify-character: 1.0.0 - micromark-util-html-tag-name: 1.1.0 - micromark-util-normalize-identifier: 1.0.0 - micromark-util-resolve-all: 1.0.0 - micromark-util-subtokenize: 1.0.2 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-factory-destination: 1.1.0 + micromark-factory-label: 1.1.0 + micromark-factory-space: 1.1.0 + micromark-factory-title: 1.1.0 + micromark-factory-whitespace: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-chunked: 1.1.0 + micromark-util-classify-character: 1.1.0 + micromark-util-html-tag-name: 1.2.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-resolve-all: 1.1.0 + micromark-util-subtokenize: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 uvu: 0.5.6 micromark-extension-gfm-autolink-literal@1.0.3: dependencies: - micromark-util-character: 1.1.0 - micromark-util-sanitize-uri: 1.1.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-util-character: 1.2.0 + micromark-util-sanitize-uri: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 uvu: 0.5.6 micromark-extension-gfm-footnote@1.0.4: dependencies: - micromark-core-commonmark: 1.0.6 - micromark-factory-space: 1.0.0 - micromark-util-character: 1.1.0 - micromark-util-normalize-identifier: 1.0.0 - micromark-util-sanitize-uri: 1.1.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-core-commonmark: 1.1.0 + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-sanitize-uri: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 uvu: 0.5.6 micromark-extension-gfm-strikethrough@1.0.4: dependencies: - micromark-util-chunked: 1.0.0 - micromark-util-classify-character: 1.0.0 - micromark-util-resolve-all: 1.0.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-util-chunked: 1.1.0 + micromark-util-classify-character: 1.1.0 + micromark-util-resolve-all: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 uvu: 0.5.6 micromark-extension-gfm-table@1.0.5: dependencies: - micromark-factory-space: 1.0.0 - micromark-util-character: 1.1.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 uvu: 0.5.6 micromark-extension-gfm-tagfilter@1.0.1: dependencies: - micromark-util-types: 1.0.2 + micromark-util-types: 1.1.0 micromark-extension-gfm-task-list-item@1.0.3: dependencies: - micromark-factory-space: 1.0.0 - micromark-util-character: 1.1.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 uvu: 0.5.6 micromark-extension-gfm@2.0.1: @@ -9706,120 +10236,126 @@ snapshots: micromark-extension-gfm-table: 1.0.5 micromark-extension-gfm-tagfilter: 1.0.1 micromark-extension-gfm-task-list-item: 1.0.3 - micromark-util-combine-extensions: 1.0.0 - micromark-util-types: 1.0.2 + micromark-util-combine-extensions: 1.1.0 + micromark-util-types: 1.1.0 - micromark-factory-destination@1.0.0: + micromark-factory-destination@1.1.0: dependencies: - micromark-util-character: 1.1.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 - micromark-factory-label@1.0.2: + micromark-factory-label@1.1.0: dependencies: - micromark-util-character: 1.1.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 uvu: 0.5.6 - micromark-factory-space@1.0.0: + micromark-factory-space@1.1.0: dependencies: - micromark-util-character: 1.1.0 - micromark-util-types: 1.0.2 + micromark-util-character: 1.2.0 + micromark-util-types: 1.1.0 - micromark-factory-title@1.0.2: + micromark-factory-title@1.1.0: dependencies: - micromark-factory-space: 1.0.0 - micromark-util-character: 1.1.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 - uvu: 0.5.6 + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 micromark-factory-whitespace@1.0.0: dependencies: - micromark-factory-space: 1.0.0 - micromark-util-character: 1.1.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + + micromark-factory-whitespace@1.1.0: + dependencies: + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 - micromark-util-character@1.1.0: + micromark-util-character@1.2.0: dependencies: - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 - micromark-util-chunked@1.0.0: + micromark-util-chunked@1.1.0: dependencies: - micromark-util-symbol: 1.0.1 + micromark-util-symbol: 1.1.0 - micromark-util-classify-character@1.0.0: + micromark-util-classify-character@1.1.0: dependencies: - micromark-util-character: 1.1.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 - micromark-util-combine-extensions@1.0.0: + micromark-util-combine-extensions@1.1.0: dependencies: - micromark-util-chunked: 1.0.0 - micromark-util-types: 1.0.2 + micromark-util-chunked: 1.1.0 + micromark-util-types: 1.1.0 - micromark-util-decode-numeric-character-reference@1.0.0: + micromark-util-decode-numeric-character-reference@1.1.0: dependencies: - micromark-util-symbol: 1.0.1 + micromark-util-symbol: 1.1.0 - micromark-util-decode-string@1.0.2: + micromark-util-decode-string@1.1.0: dependencies: decode-named-character-reference: 1.0.2 - micromark-util-character: 1.1.0 - micromark-util-decode-numeric-character-reference: 1.0.0 - micromark-util-symbol: 1.0.1 + micromark-util-character: 1.2.0 + micromark-util-decode-numeric-character-reference: 1.1.0 + micromark-util-symbol: 1.1.0 - micromark-util-encode@1.0.1: {} + micromark-util-encode@1.1.0: {} - micromark-util-html-tag-name@1.1.0: {} + micromark-util-html-tag-name@1.2.0: {} - micromark-util-normalize-identifier@1.0.0: + micromark-util-normalize-identifier@1.1.0: dependencies: - micromark-util-symbol: 1.0.1 + micromark-util-symbol: 1.1.0 - micromark-util-resolve-all@1.0.0: + micromark-util-resolve-all@1.1.0: dependencies: - micromark-util-types: 1.0.2 + micromark-util-types: 1.1.0 - micromark-util-sanitize-uri@1.1.0: + micromark-util-sanitize-uri@1.2.0: dependencies: - micromark-util-character: 1.1.0 - micromark-util-encode: 1.0.1 - micromark-util-symbol: 1.0.1 + micromark-util-character: 1.2.0 + micromark-util-encode: 1.1.0 + micromark-util-symbol: 1.1.0 - micromark-util-subtokenize@1.0.2: + micromark-util-subtokenize@1.1.0: dependencies: - micromark-util-chunked: 1.0.0 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-util-chunked: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 uvu: 0.5.6 - micromark-util-symbol@1.0.1: {} + micromark-util-symbol@1.1.0: {} - micromark-util-types@1.0.2: {} + micromark-util-types@1.1.0: {} - micromark@3.1.0: + micromark@3.2.0: dependencies: - '@types/debug': 4.1.7 - debug: 4.3.4 + '@types/debug': 4.1.12 + debug: 4.3.7 decode-named-character-reference: 1.0.2 - micromark-core-commonmark: 1.0.6 - micromark-factory-space: 1.0.0 - micromark-util-character: 1.1.0 - micromark-util-chunked: 1.0.0 - micromark-util-combine-extensions: 1.0.0 - micromark-util-decode-numeric-character-reference: 1.0.0 - micromark-util-encode: 1.0.1 - micromark-util-normalize-identifier: 1.0.0 - micromark-util-resolve-all: 1.0.0 - micromark-util-sanitize-uri: 1.1.0 - micromark-util-subtokenize: 1.0.2 - micromark-util-symbol: 1.0.1 - micromark-util-types: 1.0.2 + micromark-core-commonmark: 1.1.0 + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-chunked: 1.1.0 + micromark-util-combine-extensions: 1.1.0 + micromark-util-decode-numeric-character-reference: 1.1.0 + micromark-util-encode: 1.1.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-resolve-all: 1.1.0 + micromark-util-sanitize-uri: 1.2.0 + micromark-util-subtokenize: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 uvu: 0.5.6 transitivePeerDependencies: - supports-color @@ -9937,7 +10473,7 @@ snapshots: globby: 13.2.0 jiti: 1.18.2 mri: 1.2.0 - pathe: 1.1.1 + pathe: 1.1.2 optionalDependencies: typescript: 4.9.5 @@ -9948,6 +10484,13 @@ snapshots: pkg-types: 1.0.3 ufo: 1.1.2 + mlly@1.7.3: + dependencies: + acorn: 8.14.0 + pathe: 1.1.2 + pkg-types: 1.2.1 + ufo: 1.5.4 + mri@1.2.0: {} mrmime@1.0.1: {} @@ -10338,14 +10881,14 @@ snapshots: - supports-color - vue - nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4): + nuxt@3.6.1(@types/node@18.15.5)(encoding@0.1.13)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5): dependencies: '@nuxt/devalue': 2.0.2 '@nuxt/kit': 3.6.1(rollup@3.25.3) '@nuxt/schema': 3.6.1(rollup@3.25.3) '@nuxt/telemetry': 2.2.0(rollup@3.25.3) '@nuxt/ui-templates': 1.2.0 - '@nuxt/vite-builder': 3.6.1(@types/node@18.15.5)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@5.0.4)(vue@3.3.4) + '@nuxt/vite-builder': 3.6.1(@types/node@18.15.5)(eslint@9.0.0)(optionator@0.9.3)(rollup@3.25.3)(terser@5.17.6)(typescript@4.9.5)(vue@3.3.4) '@types/node': 18.15.5 '@unhead/ssr': 1.1.28 '@unhead/vue': 1.1.28(vue@3.3.4) @@ -10533,6 +11076,8 @@ snapshots: p-try@2.2.0: {} + package-manager-detector@0.2.2: {} + pacote@15.2.0: dependencies: '@npmcli/git': 4.0.4 @@ -10576,7 +11121,7 @@ snapshots: parse-entities@4.0.1: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 character-entities: 2.0.2 character-entities-legacy: 3.0.0 character-reference-invalid: 2.0.1 @@ -10630,6 +11175,8 @@ snapshots: dot-case: 3.0.4 tslib: 2.5.0 + path-data-parser@0.1.0: {} + path-exists@4.0.0: {} path-exists@5.0.0: {} @@ -10706,10 +11253,23 @@ snapshots: mlly: 1.4.0 pathe: 1.1.1 + pkg-types@1.2.1: + dependencies: + confbox: 0.1.8 + mlly: 1.7.3 + pathe: 1.1.2 + playwright-core@1.31.2: {} pluralize@8.0.0: {} + points-on-curve@0.2.0: {} + + points-on-path@0.2.1: + dependencies: + path-data-parser: 0.1.0 + points-on-curve: 0.2.0 + postcss-calc@9.0.1(postcss@8.4.24): dependencies: postcss: 8.4.24 @@ -11088,7 +11648,7 @@ snapshots: remark-gfm@3.0.1: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 mdast-util-gfm: 2.0.2 micromark-extension-gfm: 2.0.1 unified: 10.1.2 @@ -11099,13 +11659,13 @@ snapshots: dependencies: flat: 5.0.2 js-yaml: 4.1.0 - mdast-util-from-markdown: 1.3.0 + mdast-util-from-markdown: 1.3.1 mdast-util-to-markdown: 1.5.0 - micromark: 3.1.0 - micromark-core-commonmark: 1.0.6 - micromark-factory-space: 1.0.0 + micromark: 3.2.0 + micromark-core-commonmark: 1.1.0 + micromark-factory-space: 1.1.0 micromark-factory-whitespace: 1.0.0 - micromark-util-character: 1.1.0 + micromark-util-character: 1.2.0 parse-entities: 4.0.1 scule: 1.0.0 stringify-entities: 4.0.3 @@ -11116,8 +11676,8 @@ snapshots: remark-parse@10.0.1: dependencies: - '@types/mdast': 3.0.10 - mdast-util-from-markdown: 1.3.0 + '@types/mdast': 3.0.15 + mdast-util-from-markdown: 1.3.1 unified: 10.1.2 transitivePeerDependencies: - supports-color @@ -11125,13 +11685,13 @@ snapshots: remark-rehype@10.1.0: dependencies: '@types/hast': 2.3.4 - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 mdast-util-to-hast: 12.3.0 unified: 10.1.2 remark-squeeze-paragraphs@5.0.1: dependencies: - '@types/mdast': 3.0.10 + '@types/mdast': 3.0.15 mdast-squeeze-paragraphs: 5.2.1 unified: 10.1.2 @@ -11170,7 +11730,7 @@ snapshots: dependencies: glob: 7.2.3 - robust-predicates@3.0.1: {} + robust-predicates@3.0.2: {} rollup-plugin-dts@5.2.0(rollup@3.25.3)(typescript@4.9.5): dependencies: @@ -11193,6 +11753,13 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + roughjs@4.6.6: + dependencies: + hachure-fill: 0.5.2 + path-data-parser: 0.1.0 + points-on-curve: 0.2.0 + points-on-path: 0.2.1 + run-applescript@5.0.0: dependencies: execa: 5.1.1 @@ -11365,7 +11932,7 @@ snapshots: socket.io-client@4.6.1: dependencies: '@socket.io/component-emitter': 3.1.0 - debug: 4.3.4 + debug: 4.3.7 engine.io-client: 6.4.0 socket.io-parser: 4.2.2 transitivePeerDependencies: @@ -11376,14 +11943,14 @@ snapshots: socket.io-parser@4.2.2: dependencies: '@socket.io/component-emitter': 3.1.0 - debug: 4.3.4 + debug: 4.3.7 transitivePeerDependencies: - supports-color socks-proxy-agent@7.0.0: dependencies: agent-base: 6.0.2 - debug: 4.3.4 + debug: 4.3.7 socks: 2.7.1 transitivePeerDependencies: - supports-color @@ -11513,7 +12080,7 @@ snapshots: postcss: 8.4.24 postcss-selector-parser: 6.0.11 - stylis@4.1.3: {} + stylis@4.3.4: {} supports-color@5.5.0: dependencies: @@ -11588,6 +12155,8 @@ snapshots: tinycolor2@1.6.0: {} + tinyexec@0.3.1: {} + tinyws@0.1.0(ws@8.13.0): dependencies: ws: 8.13.0 @@ -11616,9 +12185,9 @@ snapshots: trough@2.1.0: {} - ts-api-utils@1.3.0(typescript@5.0.4): + ts-api-utils@1.3.0(typescript@4.9.5): dependencies: - typescript: 5.0.4 + typescript: 4.9.5 ts-dedent@2.2.0: {} @@ -11674,6 +12243,8 @@ snapshots: ufo@1.1.2: {} + ufo@1.5.4: {} + ultrahtml@1.2.0: {} unbuild@1.1.2: @@ -11695,7 +12266,7 @@ snapshots: mkdist: 1.1.1(typescript@4.9.5) mlly: 1.4.0 mri: 1.2.0 - pathe: 1.1.1 + pathe: 1.1.2 pkg-types: 1.0.3 pretty-bytes: 6.1.0 rollup: 3.25.3 @@ -11756,7 +12327,7 @@ snapshots: unified@10.1.2: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 bail: 2.0.2 extend: 3.0.2 is-buffer: 2.0.5 @@ -11798,7 +12369,7 @@ snapshots: unist-builder@3.0.1: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 unist-util-generated@2.0.1: {} @@ -11806,20 +12377,20 @@ snapshots: unist-util-position@4.0.4: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 unist-util-stringify-position@3.0.3: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 unist-util-visit-parents@5.1.3: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 unist-util-is: 5.2.0 unist-util-visit@4.1.2: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 unist-util-is: 5.2.0 unist-util-visit-parents: 5.1.3 @@ -11930,12 +12501,12 @@ snapshots: is-typed-array: 1.1.10 which-typed-array: 1.1.9 - uuid@9.0.0: {} + uuid@9.0.1: {} uvu@0.5.6: dependencies: dequal: 2.0.3 - diff: 5.1.0 + diff: 5.2.0 kleur: 4.1.5 sade: 1.8.1 @@ -11950,17 +12521,17 @@ snapshots: vfile-location@4.1.0: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 vfile: 5.3.7 vfile-message@3.1.4: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 unist-util-stringify-position: 3.0.3 vfile@5.3.7: dependencies: - '@types/unist': 2.0.6 + '@types/unist': 2.0.11 is-buffer: 2.0.5 unist-util-stringify-position: 3.0.3 vfile-message: 3.1.4 @@ -11968,7 +12539,7 @@ snapshots: vite-node@0.32.2(@types/node@18.15.5)(terser@5.17.6): dependencies: cac: 6.7.14 - debug: 4.3.4 + debug: 4.3.7 mlly: 1.4.0 pathe: 1.1.1 picocolors: 1.0.0 @@ -11982,7 +12553,7 @@ snapshots: - supports-color - terser - vite-plugin-checker@0.6.1(eslint@9.0.0)(optionator@0.9.3)(typescript@5.0.4)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6)): + vite-plugin-checker@0.6.1(eslint@9.0.0)(optionator@0.9.3)(typescript@4.9.5)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6)): dependencies: '@babel/code-frame': 7.21.4 ansi-escapes: 4.3.2 @@ -12005,7 +12576,7 @@ snapshots: optionalDependencies: eslint: 9.0.0 optionator: 0.9.3 - typescript: 5.0.4 + typescript: 4.9.5 vite-plugin-inspect@0.7.29(rollup@3.25.3)(vite@4.3.9(@types/node@18.15.5)(terser@5.17.6)): dependencies: @@ -12048,6 +12619,8 @@ snapshots: vscode-jsonrpc@6.0.0: {} + vscode-jsonrpc@8.2.0: {} + vscode-languageclient@7.0.0: dependencies: minimatch: 3.1.2 @@ -12059,16 +12632,31 @@ snapshots: vscode-jsonrpc: 6.0.0 vscode-languageserver-types: 3.16.0 + vscode-languageserver-protocol@3.17.5: + dependencies: + vscode-jsonrpc: 8.2.0 + vscode-languageserver-types: 3.17.5 + + vscode-languageserver-textdocument@1.0.12: {} + vscode-languageserver-textdocument@1.0.8: {} vscode-languageserver-types@3.16.0: {} + vscode-languageserver-types@3.17.5: {} + vscode-languageserver@7.0.0: dependencies: vscode-languageserver-protocol: 3.16.0 + vscode-languageserver@9.0.1: + dependencies: + vscode-languageserver-protocol: 3.17.5 + vscode-uri@3.0.7: {} + vscode-uri@3.0.8: {} + vue-bundle-renderer@1.0.3: dependencies: ufo: 1.1.2 @@ -12088,7 +12676,7 @@ snapshots: vue-eslint-parser@9.4.2(eslint@9.0.0): dependencies: - debug: 4.3.4 + debug: 4.3.7 eslint: 9.0.0 eslint-scope: 7.2.0 eslint-visitor-keys: 3.4.1 @@ -12135,7 +12723,7 @@ snapshots: web-streams-polyfill@3.2.1: {} - web-worker@1.2.0: {} + web-worker@1.3.0: {} webidl-conversions@3.0.1: {} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d90d32f6d..9a1dea873 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: version: ${{ steps.genver.outputs.version }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -40,13 +40,14 @@ jobs: - Adaptors/MongoDB/tests - Adaptors/Memory/tests - Adaptors/S3/tests + - Adaptors/Embed/tests os: - ubuntu-latest fail-fast: false runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -54,6 +55,11 @@ jobs: - name: Install AWSCLI (the one in the Github runner does not work) run: | pip install awscli + + - name: Setup Terraform + uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + with: + terraform_version: latest - name: Setup just run: | @@ -67,12 +73,12 @@ jobs: - name: Dotnet Restore run: | - MONITOR_PREFIX="monitor/restore/" MONITOR_CD=${{ matrix.projects }} tools/retry.sh -w 60 -- tools/monitor.sh \ + MONITOR_PREFIX="monitor/restore/" tools/retry.sh -w 60 -- tools/monitor.sh \ dotnet restore - name: Dotnet Build run: | - MONITOR_PREFIX="monitor/build/" MONITOR_CD=${{ matrix.projects }} tools/monitor.sh \ + MONITOR_PREFIX="monitor/build/" tools/monitor.sh \ dotnet build - name: Run tests @@ -113,7 +119,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -121,7 +127,12 @@ jobs: - name: Install AWSCLI (the one in the Github runner does not work) run: | pip install awscli - + + - name: Setup Terraform + uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + with: + terraform_version: latest + - name: Setup just run: | sudo snap install --edge --classic just @@ -133,12 +144,12 @@ jobs: - name: Dotnet Restore run: | - MONITOR_PREFIX="monitor/restore/" MONITOR_CD=${{ matrix.projects }} tools/retry.sh -w 60 -- tools/monitor.sh \ + MONITOR_PREFIX="monitor/restore/" tools/retry.sh -w 60 -- tools/monitor.sh \ dotnet restore - name: Dotnet Build run: | - MONITOR_PREFIX="monitor/build/" MONITOR_CD=${{ matrix.projects }} tools/monitor.sh \ + MONITOR_PREFIX="monitor/build/" tools/monitor.sh \ dotnet build - name: Run tests @@ -169,24 +180,21 @@ jobs: - Common/tests - Adaptors/MongoDB/tests - Adaptors/Memory/tests + - Adaptors/Embed/tests fail-fast: false runs-on: windows-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true - name: Dotnet Restore - run: | - cd ${{ matrix.projects }} - dotnet restore + run: dotnet restore - name: Dotnet Build - run: | - cd ${{ matrix.projects }} - dotnet build + run: dotnet build - name: Run tests run: | @@ -205,7 +213,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -222,7 +230,7 @@ jobs: VERSION: ${{ needs.versionning.outputs.version }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} @@ -258,7 +266,7 @@ jobs: - worker=htcmock buildWorker steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} @@ -306,7 +314,7 @@ jobs: username: ${{ secrets.DOCKER_HUB_LOGIN }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Analyze for critical and high CVEs - uses: docker/scout-action@cc6bf8dd03587425ef920278b3e2726ba8d791e8 # v1 + uses: docker/scout-action@b23590dc1e4d09febc00cfcbc51e9e8c0f7ee9f3 # v1 with: command: cves image: "${{ matrix.image }}:${{ needs.versionning.outputs.version }}" @@ -336,7 +344,7 @@ jobs: - Verbose steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -351,6 +359,8 @@ jobs: - name: Setup Terraform uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + with: + terraform_version: latest - name: Deploy Core run: | @@ -440,12 +450,18 @@ jobs: - { queue: pubsub, object: minio, log-level: Information, cinit: true } - { queue: sqs, object: minio, log-level: Information, cinit: true } + - { queue: activemq, object: embed, log-level: Information, cinit: true } + - { queue: rabbitmq, object: embed, log-level: Information, cinit: true } + - { queue: rabbitmq091, object: embed, log-level: Information, cinit: true } + - { queue: pubsub, object: embed, log-level: Information, cinit: true } + - { queue: sqs, object: embed, log-level: Information, cinit: true } + - { queue: activemq, object: redis, log-level: Information, cinit: false } name: HtcMock ${{ matrix.target.queue }} ${{ matrix.target.object }} ${{ matrix.target.log-level }} ${{ matrix.target.cinit }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -457,9 +473,11 @@ jobs: - name: Setup just run: | sudo snap install --edge --classic just - + - name: Setup Terraform uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + with: + terraform_version: latest - name: Deploy Core run: | @@ -604,13 +622,17 @@ jobs: name: Docker Windows steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true - - name: Setup Terraform - run: choco install terraform mongodb-shell 7zip just + - name: Setup Terraform and other tools + run: | + choco install terraform --version=1.10.1 -y + choco install mongodb-shell -y + choco install 7zip -y + choco install just -y - name: Setup AWS cli uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4 @@ -623,6 +645,11 @@ jobs: run: just -v tag=$env:VERSION object=local worker=htcmock ingress=false prometheus=false grafana=false seq=false queue=rabbitmq091 deploy shell: powershell + - name: Pull image + run: docker pull dockerhubaneo/armonik_core_htcmock_test_client:$env:VERSION + timeout-minutes: 10 + shell: powershell + - name: Run HtcMock test 100 tasks 1 level timeout-minutes: 3 shell: powershell @@ -724,7 +751,7 @@ jobs: name: "Test connectivity - Ca installed ${{ matrix.ca }}" steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -739,6 +766,8 @@ jobs: - name: Setup Terraform uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + with: + terraform_version: latest - name: Deploy Core run: | @@ -808,7 +837,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -823,6 +852,8 @@ jobs: - name: Setup Terraform uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + with: + terraform_version: latest - name: Deploy Core run: | @@ -926,7 +957,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -941,6 +972,8 @@ jobs: - name: Setup Terraform uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + with: + terraform_version: latest - name: Deploy Core run: | @@ -998,7 +1031,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -1013,6 +1046,8 @@ jobs: - name: Setup Terraform uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + with: + terraform_version: latest - name: Deploy Core run: | @@ -1068,7 +1103,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -1083,6 +1118,8 @@ jobs: - name: Setup Terraform uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + with: + terraform_version: latest - name: Deploy Core run: | diff --git a/.github/workflows/code-formatting.yml b/.github/workflows/code-formatting.yml index d6376a6b7..54febaccc 100644 --- a/.github/workflows/code-formatting.yml +++ b/.github/workflows/code-formatting.yml @@ -9,12 +9,12 @@ jobs: timeout-minutes: 15 steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} - name: Install .NET Core - uses: actions/setup-dotnet@6bd8b7f7774af54e05809fcc5431931b3eb1ddee # v4 + uses: actions/setup-dotnet@3e891b0cb619bf60e2c25674b222b8940e2c1c25 # v4 with: dotnet-version: 8.x @@ -50,7 +50,7 @@ jobs: run: | git diff > patch-csharp.diff - - uses: actions/upload-artifact@84480863f228bb9747b473957fcc9e309aa96097 # v4 + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 if: ${{ failure() && steps.check-diff.conclusion == 'failure' }} with: name: patch-csharp @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} @@ -72,7 +72,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index c234d3b3d..653a16274 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -21,12 +21,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - run: npm i -g pnpm @antfu/ni - name: Setup Node - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4 + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 with: node-version: "18" cache: "pnpm" diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml index 33358768e..502714f54 100644 --- a/.github/workflows/make-release.yml +++ b/.github/workflows/make-release.yml @@ -16,7 +16,7 @@ jobs: release: ${{ steps.release.outputs.version }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -81,7 +81,7 @@ jobs: - worker=htcmock buildWorker steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} @@ -107,7 +107,7 @@ jobs: VERSION: ${{ needs.versionning.outputs.release }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} @@ -128,11 +128,11 @@ jobs: needs: - versionning steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: fetch-depth: 0 - - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4 + - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 with: node-version: 18 diff --git a/.github/workflows/manual-integration-test.yml b/.github/workflows/manual-integration-test.yml index 903249d60..c700104a9 100644 --- a/.github/workflows/manual-integration-test.yml +++ b/.github/workflows/manual-integration-test.yml @@ -10,7 +10,7 @@ jobs: version: ${{ steps.genver.outputs.version }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -73,7 +73,7 @@ jobs: find /tmp/armoniklogs -name "*.log" - name: Store logs - uses: actions/upload-artifact@84480863f228bb9747b473957fcc9e309aa96097 # v4 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 if: always() with: name: Logs Stream @@ -88,7 +88,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -124,7 +124,7 @@ jobs: find /tmp/armoniklogs -name "*.log" - name: Store logs - uses: actions/upload-artifact@84480863f228bb9747b473957fcc9e309aa96097 # v4 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 if: always() with: name: Logs HtcMock diff --git a/.github/workflows/manual-test.yml b/.github/workflows/manual-test.yml index be41aeca5..f8daa0f9f 100644 --- a/.github/workflows/manual-test.yml +++ b/.github/workflows/manual-test.yml @@ -23,7 +23,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -92,7 +92,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true @@ -154,7 +154,7 @@ jobs: runs-on: windows-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.ref }} submodules: true diff --git a/.github/workflows/scout-cron.yml b/.github/workflows/scout-cron.yml index 64c0835c0..57dbc3c8e 100644 --- a/.github/workflows/scout-cron.yml +++ b/.github/workflows/scout-cron.yml @@ -3,6 +3,7 @@ name: Scout Images on: schedule: - cron: '10 9 * * 1' # 9:10 on Monday + workflow_dispatch: jobs: releases: @@ -10,6 +11,7 @@ jobs: outputs: releases: ${{ steps.releases.outputs.releases }} steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: List last releases id: releases run: echo "releases:$(gh release list -L 3 --json tagName -q '[ .[].tagName ]')" >> "$GITHUB_OUTPUT" @@ -41,7 +43,7 @@ jobs: username: ${{ secrets.DOCKER_HUB_LOGIN }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Analyze for critical and high CVEs - uses: docker/scout-action@cc6bf8dd03587425ef920278b3e2726ba8d791e8 # v1 + uses: docker/scout-action@b23590dc1e4d09febc00cfcbc51e9e8c0f7ee9f3 # v1 with: command: cves image: "${{ matrix.image }}:${{ matrix.image }}" diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml index b1e1efda3..79974443c 100644 --- a/.github/workflows/sonar.yml +++ b/.github/workflows/sonar.yml @@ -15,24 +15,24 @@ jobs: runs-on: windows-latest steps: - name: Set up JDK - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 + uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 with: distribution: 'zulu' java-version: 17 - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis submodules: true - name: Cache SonarCloud packages - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4 with: path: .\.sonar\cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - name: Cache SonarCloud scanner id: cache-sonar-scanner - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4 with: path: .\.sonar\scanner key: ${{ runner.os }}-sonar-scanner diff --git a/.github/workflows/validate-docs-generation.yml b/.github/workflows/validate-docs-generation.yml index 7e863e57a..48c5c4bc9 100644 --- a/.github/workflows/validate-docs-generation.yml +++ b/.github/workflows/validate-docs-generation.yml @@ -15,12 +15,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - run: npm install -g pnpm@^9.0.2 - name: Setup Node - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4 + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 with: node-version: 18 cache: "pnpm" @@ -40,11 +40,11 @@ jobs: name: Lint Markdown runs-on: ubuntu-latest steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - run: npm install -g pnpm@^9.0.2 - - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4 + - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 with: node-version: 18 cache: pnpm @@ -58,11 +58,11 @@ jobs: name: Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - run: npm install -g pnpm@^9.0.2 - - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4 + - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 with: node-version: 18 cache: pnpm diff --git a/Adaptors/Amqp/src/ArmoniK.Core.Adapters.Amqp.csproj b/Adaptors/Amqp/src/ArmoniK.Core.Adapters.Amqp.csproj index 0673b4f09..6e306fa25 100644 --- a/Adaptors/Amqp/src/ArmoniK.Core.Adapters.Amqp.csproj +++ b/Adaptors/Amqp/src/ArmoniK.Core.Adapters.Amqp.csproj @@ -26,12 +26,12 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + false runtime diff --git a/Adaptors/Amqp/tests/ArmoniK.Core.Adapters.Amqp.Tests.csproj b/Adaptors/Amqp/tests/ArmoniK.Core.Adapters.Amqp.Tests.csproj index 33f39617c..7ddc6f617 100644 --- a/Adaptors/Amqp/tests/ArmoniK.Core.Adapters.Amqp.Tests.csproj +++ b/Adaptors/Amqp/tests/ArmoniK.Core.Adapters.Amqp.Tests.csproj @@ -20,7 +20,7 @@ - + diff --git a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj new file mode 100644 index 000000000..05f2744e5 --- /dev/null +++ b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj @@ -0,0 +1,74 @@ + + + net8.0 + win-x64;linux-x64;linux-arm64 + True + ANEO + Copyright (C) ANEO, 2021-2022 + AGPL-3.0-or-later + True + true + enable + true + + + + Embedded + true + DEBUG;TRACE + + + + true + true + snupkg + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + runtime + + + runtime + + + runtime + + + false + runtime + + + runtime + + + runtime + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + runtime + + + + + + false + runtime + + + false + runtime + + + + diff --git a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings new file mode 100644 index 000000000..89316e414 --- /dev/null +++ b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings @@ -0,0 +1,2 @@ + + Library \ No newline at end of file diff --git a/Adaptors/MongoDB/src/Options/QueueStorage.cs b/Adaptors/Embed/src/ObjectBuilder.cs similarity index 51% rename from Adaptors/MongoDB/src/Options/QueueStorage.cs rename to Adaptors/Embed/src/ObjectBuilder.cs index 8ca8cb351..28f7063ef 100644 --- a/Adaptors/MongoDB/src/Options/QueueStorage.cs +++ b/Adaptors/Embed/src/ObjectBuilder.cs @@ -1,4 +1,4 @@ -// This file is part of the ArmoniK project +// This file is part of the ArmoniK project // // Copyright (C) ANEO, 2021-2024. All rights reserved. // @@ -15,20 +15,27 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using System; +using ArmoniK.Core.Base; +using ArmoniK.Core.Utils; using JetBrains.Annotations; -namespace ArmoniK.Core.Adapters.MongoDB.Options; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +namespace ArmoniK.Core.Adapters.Embed; + +/// +/// Class for building Embed instance and Object interfaces through Dependency Injection +/// [PublicAPI] -public class QueueStorage +public class ObjectBuilder : IDependencyInjectionBuildable { - public const string SettingSection = nameof(MongoDB) + ":" + nameof(QueueStorage); - - public TimeSpan LockRefreshPeriodicity { get; set; } = TimeSpan.FromMinutes(2); - - public TimeSpan PollPeriodicity { get; set; } = TimeSpan.FromSeconds(5); - - public TimeSpan LockRefreshExtension { get; set; } = TimeSpan.FromMinutes(5); + /// + [PublicAPI] + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) + => serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); } diff --git a/Adaptors/Embed/src/ObjectStorage.cs b/Adaptors/Embed/src/ObjectStorage.cs new file mode 100644 index 000000000..8bb28d693 --- /dev/null +++ b/Adaptors/Embed/src/ObjectStorage.cs @@ -0,0 +1,86 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +namespace ArmoniK.Core.Adapters.Embed; + +[UsedImplicitly] +public class ObjectStorage : IObjectStorage +{ + private readonly ILogger logger_; + private bool isInitialized_; + + /// + /// implementation for Redis + /// + /// Logger used to print logs + public ObjectStorage(ILogger logger) + => logger_ = logger; + + /// + public Task Init(CancellationToken cancellationToken) + { + isInitialized_ = true; + return Task.CompletedTask; + } + + /// + public Task Check(HealthCheckTag tag) + => Task.FromResult(isInitialized_ + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy("Object storage not initialized yet.")); + + /// + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + { + var array = new List(); + + await foreach (var val in valueChunks.WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + array.AddRange(val.Span); + } + + return (array.ToArray(), array.Count); + } + + /// + public IAsyncEnumerable GetValuesAsync(byte[] id, + CancellationToken cancellationToken = default) + => AsyncEnumerable.Repeat(id, + 1); + + /// + public Task TryDeleteAsync(IEnumerable ids, + CancellationToken cancellationToken = default) + => Task.CompletedTask; +} diff --git a/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj new file mode 100644 index 000000000..975cca170 --- /dev/null +++ b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj @@ -0,0 +1,28 @@ + + + net8.0 + win-x64;linux-x64;linux-arm64 + false + enable + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + diff --git a/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings new file mode 100644 index 000000000..89316e414 --- /dev/null +++ b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings @@ -0,0 +1,2 @@ + + Library \ No newline at end of file diff --git a/Adaptors/Embed/tests/ObjectStorageTests.cs b/Adaptors/Embed/tests/ObjectStorageTests.cs new file mode 100644 index 000000000..472d36a4a --- /dev/null +++ b/Adaptors/Embed/tests/ObjectStorageTests.cs @@ -0,0 +1,78 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System.Collections.Generic; +using System.IO; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; +using ArmoniK.Core.Common.Tests.TestBase; +using ArmoniK.Core.Common.Utils; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +using NUnit.Framework; + +namespace ArmoniK.Core.Adapters.Embed.Tests; + +[TestFixture] +public class ObjectStorageTests : ObjectStorageTestBase +{ + private static readonly string SolutionRoot = + Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(typeof(ObjectStorageTests) + .Assembly + .Location))))) ?? + string.Empty)); + + private static readonly string S3Path = + $"{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.Embed.dll"; + + + protected override void GetObjectStorageInstance() + { + Dictionary minimalConfig = new() + { + { + "Components:ObjectStorageAdaptorSettings:ClassName", "ArmoniK.Core.Adapters.Embed.ObjectBuilder" + }, + { + "Components:ObjectStorageAdaptorSettings:AdapterAbsolutePath", $"{SolutionRoot}{S3Path}" + }, + }; + + var configuration = new ConfigurationManager(); + configuration.AddInMemoryCollection(minimalConfig); + + var services = new ServiceCollection(); + services.AddLogging(); + var logger = new LoggerInit(configuration); + + services.AddAdapter(configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()); + + var provider = services.BuildServiceProvider(new ServiceProviderOptions + { + ValidateOnBuild = true, + }); + + ObjectStorage = provider.GetRequiredService(); + RunTests = true; + } +} diff --git a/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj b/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj index 9c88183b8..f369c28fe 100644 --- a/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj +++ b/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj @@ -1,4 +1,4 @@ - + net8.0 win-x64;linux-x64;linux-arm64 @@ -10,10 +10,18 @@ True true enable + true - + + false + runtime + + + false + runtime + diff --git a/Adaptors/LocalStorage/src/ServiceCollectionExt.cs b/Adaptors/LocalStorage/src/ObjectBuilder.cs similarity index 68% rename from Adaptors/LocalStorage/src/ServiceCollectionExt.cs rename to Adaptors/LocalStorage/src/ObjectBuilder.cs index fb69ba594..7de022ad8 100644 --- a/Adaptors/LocalStorage/src/ServiceCollectionExt.cs +++ b/Adaptors/LocalStorage/src/ObjectBuilder.cs @@ -15,9 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Common.Injection.Options; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base; using ArmoniK.Core.Utils; using JetBrains.Annotations; @@ -28,34 +26,27 @@ namespace ArmoniK.Core.Adapters.LocalStorage; -public static class ServiceCollectionExt +/// +/// Class for building Local Storage instance and Object interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable { + /// [PublicAPI] - public static IServiceCollection AddLocalStorage(this IServiceCollection serviceCollection, - ConfigurationManager configuration, - ILogger logger) + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) { - var components = configuration.GetSection(Components.SettingSection); - - if (components["ObjectStorage"] != "ArmoniK.Adapters.LocalStorage.ObjectStorage") - { - return serviceCollection; - } - serviceCollection.AddOption(configuration, Options.LocalStorage.SettingSection, out Options.LocalStorage storageOptions); - using var _ = logger.BeginNamedScope("Object Local configuration", - ("Path", storageOptions.Path)); - logger.LogDebug("setup local storage"); serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage), sp => new ObjectStorage(storageOptions.Path, storageOptions.ChunkSize, sp.GetRequiredService>())); - - return serviceCollection; } } diff --git a/Adaptors/LocalStorage/src/ObjectStorage.cs b/Adaptors/LocalStorage/src/ObjectStorage.cs index 43607022c..7ef35013f 100644 --- a/Adaptors/LocalStorage/src/ObjectStorage.cs +++ b/Adaptors/LocalStorage/src/ObjectStorage.cs @@ -19,13 +19,13 @@ using System.Collections.Generic; using System.IO; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; -using ArmoniK.Api.Common.Utils; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base.Exceptions; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -94,15 +94,16 @@ public Task Check(HealthCheckTag tag) }; /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { long size = 0; + var key = Guid.NewGuid() + .ToString(); var filename = Path.Combine(path_, key); - using var _ = logger_.LogFunction(filename); // Write to temporary file await using var file = File.Open(filename, @@ -135,18 +136,18 @@ public async Task AddOrUpdateAsync(string await file.FlushAsync(cancellationToken) .ConfigureAwait(false); - return size; + return (Encoding.UTF8.GetBytes(key), size); } /// - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + var key = Encoding.UTF8.GetString(id); + var filename = Path.Combine(path_, key); - using var _ = logger_.LogFunction(filename); - if (!File.Exists(filename)) { throw new ObjectDataNotFoundException($"The object {key} has not been found in {path_}"); @@ -187,29 +188,24 @@ public async IAsyncEnumerable GetValuesAsync(string } /// - public async Task TryDeleteAsync(IEnumerable keys, + public async Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) { - foreach (var key in keys) + foreach (var id in ids) { + var key = Encoding.UTF8.GetString(id); await TryDeleteAsync(key, cancellationToken) .ConfigureAwait(false); } } - /// - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); - public Task TryDeleteAsync(string key, CancellationToken cancellationToken = default) { var filename = Path.Combine(path_, key); - using var _ = logger_.LogFunction(filename); - File.Delete(filename); return Task.FromResult(true); diff --git a/Adaptors/LocalStorage/tests/ArmoniK.Core.Adapters.LocalStorage.Tests.csproj b/Adaptors/LocalStorage/tests/ArmoniK.Core.Adapters.LocalStorage.Tests.csproj index 58ff1bc50..fc976a8ab 100644 --- a/Adaptors/LocalStorage/tests/ArmoniK.Core.Adapters.LocalStorage.Tests.csproj +++ b/Adaptors/LocalStorage/tests/ArmoniK.Core.Adapters.LocalStorage.Tests.csproj @@ -13,7 +13,7 @@ - + diff --git a/Adaptors/Memory/src/ObjectStorage.cs b/Adaptors/Memory/src/ObjectStorage.cs new file mode 100644 index 000000000..1e91b7e52 --- /dev/null +++ b/Adaptors/Memory/src/ObjectStorage.cs @@ -0,0 +1,104 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; +using ArmoniK.Utils; + +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace ArmoniK.Core.Adapters.Memory; + +public class ObjectStorage : IObjectStorage +{ + private readonly ConcurrentDictionary store_ = new(); + private bool isInitialized_; + + /// + public Task Init(CancellationToken cancellationToken) + { + isInitialized_ = true; + return Task.CompletedTask; + } + + /// + public Task Check(HealthCheckTag tag) + => Task.FromResult(isInitialized_ + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy()); + + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + { + var array = new List(); + + var key = Guid.NewGuid() + .ToString(); + + await foreach (var val in valueChunks.WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + array.AddRange(val.ToArray()); + } + + store_[key] = array.ToArray(); + + return (Encoding.UTF8.GetBytes(key), array.Count); + } + +#pragma warning disable CS1998 + public async IAsyncEnumerable GetValuesAsync(byte[] id, +#pragma warning restore CS1998 + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var key = Encoding.UTF8.GetString(id); + if (!store_.TryGetValue(key, + out var value)) + { + throw new ObjectDataNotFoundException(); + } + + foreach (var chunk in value.ToChunks(100)) + { + yield return chunk; + } + } + + public Task TryDeleteAsync(IEnumerable ids, + CancellationToken cancellationToken = default) + { + foreach (var id in ids) + { + var key = Encoding.UTF8.GetString(id); + + store_.TryRemove(key, + out _); + } + + return Task.CompletedTask; + } +} diff --git a/Adaptors/Memory/src/ResultTable.cs b/Adaptors/Memory/src/ResultTable.cs index 9934fe0e3..dd93aa36d 100644 --- a/Adaptors/Memory/src/ResultTable.cs +++ b/Adaptors/Memory/src/ResultTable.cs @@ -25,6 +25,7 @@ using System.Threading.Tasks; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; diff --git a/Adaptors/Memory/src/TaskTable.cs b/Adaptors/Memory/src/TaskTable.cs index 4433b7556..b9e5553fb 100644 --- a/Adaptors/Memory/src/TaskTable.cs +++ b/Adaptors/Memory/src/TaskTable.cs @@ -271,7 +271,7 @@ public Task RemoveRemainingDataDependenciesAsync(ICollection taskIds, { var remainingDep = data.RemainingDataDependencies; - foreach (var dep in dependenciesToRemove.Select(TaskData.EscapeKey)) + foreach (var dep in dependenciesToRemove) { remainingDep.Remove(dep); } diff --git a/Adaptors/Memory/tests/ArmoniK.Core.Adapters.Memory.Tests.csproj b/Adaptors/Memory/tests/ArmoniK.Core.Adapters.Memory.Tests.csproj index 5ed591d41..b759b347e 100644 --- a/Adaptors/Memory/tests/ArmoniK.Core.Adapters.Memory.Tests.csproj +++ b/Adaptors/Memory/tests/ArmoniK.Core.Adapters.Memory.Tests.csproj @@ -20,7 +20,7 @@ - + diff --git a/Adaptors/MongoDB/tests/ObjectStorageTests.cs b/Adaptors/Memory/tests/ObjectStorageTests.cs similarity index 68% rename from Adaptors/MongoDB/tests/ObjectStorageTests.cs rename to Adaptors/Memory/tests/ObjectStorageTests.cs index 0ac4b90ec..2a813a797 100644 --- a/Adaptors/MongoDB/tests/ObjectStorageTests.cs +++ b/Adaptors/Memory/tests/ObjectStorageTests.cs @@ -15,14 +15,11 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Common.Tests.TestBase; -using Microsoft.Extensions.DependencyInjection; - using NUnit.Framework; -namespace ArmoniK.Core.Adapters.MongoDB.Tests; +namespace ArmoniK.Core.Adapters.Memory.Tests; [TestFixture] public class ObjectStorageTests : ObjectStorageTestBase @@ -30,17 +27,13 @@ public class ObjectStorageTests : ObjectStorageTestBase public override void TearDown() { ObjectStorage = null; - tableProvider_?.Dispose(); - RunTests = false; + RunTests = false; } - private MongoDatabaseProvider? tableProvider_; protected override void GetObjectStorageInstance() { - tableProvider_ = new MongoDatabaseProvider(serviceConfigurator: collection => collection.AddSingleton()); - var provider = tableProvider_.GetServiceProvider(); - ObjectStorage = provider.GetRequiredService(); + ObjectStorage = new ObjectStorage(); RunTests = true; } } diff --git a/Adaptors/MongoDB/src/ArmoniK.Core.Adapters.MongoDB.csproj b/Adaptors/MongoDB/src/ArmoniK.Core.Adapters.MongoDB.csproj index 0dab58955..2f19ddee3 100644 --- a/Adaptors/MongoDB/src/ArmoniK.Core.Adapters.MongoDB.csproj +++ b/Adaptors/MongoDB/src/ArmoniK.Core.Adapters.MongoDB.csproj @@ -1,4 +1,4 @@ - + @@ -32,6 +32,7 @@ + diff --git a/Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs b/Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs deleted file mode 100644 index fa9560aeb..000000000 --- a/Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs +++ /dev/null @@ -1,85 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.Threading.Tasks; - -using ArmoniK.Core.Adapters.MongoDB.Common; - -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; - -namespace ArmoniK.Core.Adapters.MongoDB.Object; - -public class ObjectDataModelMapping : IMongoDataModelMapping -{ - public const string Collection = "Object"; - - [BsonId] - public string Id - => $"{Key}.{ChunkIdx}"; - - [BsonElement] - public string Key { get; set; } = ""; - - [BsonElement] - public byte[] Chunk { get; set; } = Array.Empty(); - - [BsonElement] - public int ChunkIdx { get; set; } - - /// - [BsonIgnore] - public string CollectionName { get; } = Collection; - - /// - public async Task InitializeIndexesAsync(IClientSessionHandle sessionHandle, - IMongoCollection collection, - Options.MongoDB options) - { - var keyIndex = Builders.IndexKeys.Hashed(model => model.Key); - var chunkIdxIndex = Builders.IndexKeys.Hashed(model => model.ChunkIdx); - var iDIndex = Builders.IndexKeys.Hashed(model => model.Id); - - var indexModels = new CreateIndexModel[] - { - new(iDIndex, - new CreateIndexOptions - { - Name = nameof(iDIndex), - }), - new(keyIndex, - new CreateIndexOptions - { - Name = nameof(keyIndex), - }), - new(chunkIdxIndex, - new CreateIndexOptions - { - Name = nameof(chunkIdxIndex), - }), - }; - - await collection.Indexes.CreateManyAsync(sessionHandle, - indexModels) - .ConfigureAwait(false); - } - - public Task ShardCollectionAsync(IClientSessionHandle sessionHandle, - Options.MongoDB options) - => Task.CompletedTask; -} diff --git a/Adaptors/MongoDB/src/ObjectStorage.cs b/Adaptors/MongoDB/src/ObjectStorage.cs deleted file mode 100644 index 0b84fcee9..000000000 --- a/Adaptors/MongoDB/src/ObjectStorage.cs +++ /dev/null @@ -1,199 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; - -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Adapters.MongoDB.Common; -using ArmoniK.Core.Adapters.MongoDB.Object; -using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; -using ArmoniK.Utils; - -using JetBrains.Annotations; - -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Logging; - -using MongoDB.Driver; -using MongoDB.Driver.Linq; - -namespace ArmoniK.Core.Adapters.MongoDB; - -[PublicAPI] -public class ObjectStorage : IObjectStorage -{ - private readonly ILogger logger_; - private readonly MongoCollectionProvider objectCollectionProvider_; - private readonly string objectStorageName_; - private readonly SessionProvider sessionProvider_; - private bool isInitialized_; - - public ObjectStorage(SessionProvider sessionProvider, - MongoCollectionProvider objectCollectionProvider, - ILogger logger, - Options.ObjectStorage options) - { - if (options.ChunkSize == 0) - { - throw new ArgumentOutOfRangeException(nameof(options), - $"Minimum value for {nameof(Options.ObjectStorage.ChunkSize)} is 1."); - } - - sessionProvider_ = sessionProvider; - objectCollectionProvider_ = objectCollectionProvider; - objectStorageName_ = "storage/"; - ChunkSize = options.ChunkSize; - logger_ = logger; - } - - public int ChunkSize { get; } - - /// - public async Task Init(CancellationToken cancellationToken) - { - if (!isInitialized_) - { - await sessionProvider_.Init(cancellationToken) - .ConfigureAwait(false); - sessionProvider_.Get(); - - await objectCollectionProvider_.Init(cancellationToken) - .ConfigureAwait(false); - objectCollectionProvider_.Get(); - } - - isInitialized_ = true; - } - - /// - public Task Check(HealthCheckTag tag) - => Task.FromResult(isInitialized_ - ? HealthCheckResult.Healthy() - : HealthCheckResult.Unhealthy()); - - /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) - { - long size = 0; - var dbKey = objectStorageName_ + key; - using var _ = logger_.LogFunction(dbKey); - var objectCollection = objectCollectionProvider_.Get(); - - var taskList = new List(); - - var idx = 0; - await foreach (var chunk in valueChunks.WithCancellation(cancellationToken) - .ConfigureAwait(false)) - { - size += chunk.Length; - taskList.Add(objectCollection.InsertOneAsync(new ObjectDataModelMapping - { - Chunk = chunk.ToArray(), - ChunkIdx = idx, - Key = dbKey, - }, - cancellationToken: cancellationToken)); - ++idx; - } - - // If there was no chunks, add an empty chunk, just so that it could be found in the future - if (idx == 0) - { - taskList.Add(objectCollection.InsertOneAsync(new ObjectDataModelMapping - { - Chunk = Array.Empty(), - ChunkIdx = idx, - Key = dbKey, - }, - cancellationToken: cancellationToken)); - } - - await taskList.WhenAll() - .ConfigureAwait(false); - - return size; - } - - /// - async IAsyncEnumerable IObjectStorage.GetValuesAsync(string key, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - using var _ = logger_.LogFunction(objectStorageName_ + key); - var sessionHandle = sessionProvider_.Get(); - var objectCollection = objectCollectionProvider_.Get(); - - var throwException = true; - await foreach (var chunk in objectCollection.AsQueryable(sessionHandle) - .Where(odm => odm.Key == objectStorageName_ + key) - .OrderBy(odm => odm.ChunkIdx) - .Select(odm => odm.Chunk) - .ToAsyncEnumerable(cancellationToken) - .ConfigureAwait(false)) - { - throwException = false; - yield return chunk; - } - - if (throwException) - { - throw new ObjectDataNotFoundException($"Result {key} not found"); - } - } - - /// - public async Task TryDeleteAsync(IEnumerable keys, - CancellationToken cancellationToken = default) - - { - using var _ = logger_.LogFunction(objectStorageName_); - var objectCollection = objectCollectionProvider_.Get(); - - var names = keys.Select(key => objectStorageName_ + key); - - await objectCollection.DeleteManyAsync(odm => names.Contains(odm.Key), - cancellationToken) - .ConfigureAwait(false); - logger_.LogInformation("Deleted data with {resultIds}", - keys); - } - - /// - public async IAsyncEnumerable ListKeysAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) - { - using var _ = logger_.LogFunction(); - var sessionHandle = sessionProvider_.Get(); - var objectCollection = objectCollectionProvider_.Get(); - - await foreach (var key in objectCollection.AsQueryable(sessionHandle) - .Where(odm => odm.ChunkIdx == 0) - .Select(odm => odm.Key) - .ToAsyncEnumerable(cancellationToken) - .ConfigureAwait(false)) - { - yield return key; - } - } -} diff --git a/Adaptors/MongoDB/src/Options/MongoDB.cs b/Adaptors/MongoDB/src/Options/MongoDB.cs index dcbf8dfc7..403112a6d 100644 --- a/Adaptors/MongoDB/src/Options/MongoDB.cs +++ b/Adaptors/MongoDB/src/Options/MongoDB.cs @@ -56,10 +56,6 @@ public class MongoDB public TableStorage TableStorage { get; set; } = new(); - public ObjectStorage ObjectStorage { get; set; } = new(); - - public QueueStorage QueueStorage { get; set; } = new(); - public int MaxConnectionPoolSize { get; set; } = 500; public TimeSpan ServerSelectionTimeout { get; set; } = TimeSpan.FromMinutes(2); diff --git a/Adaptors/MongoDB/src/ResultTable.cs b/Adaptors/MongoDB/src/ResultTable.cs index a4de3c194..b0a4f357e 100644 --- a/Adaptors/MongoDB/src/ResultTable.cs +++ b/Adaptors/MongoDB/src/ResultTable.cs @@ -27,6 +27,7 @@ using ArmoniK.Core.Adapters.MongoDB.Common; using ArmoniK.Core.Adapters.MongoDB.Table.DataModel; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; diff --git a/Adaptors/MongoDB/src/ServiceCollectionExt.cs b/Adaptors/MongoDB/src/ServiceCollectionExt.cs index 9fa9e90a7..a38fd6e3d 100644 --- a/Adaptors/MongoDB/src/ServiceCollectionExt.cs +++ b/Adaptors/MongoDB/src/ServiceCollectionExt.cs @@ -77,14 +77,6 @@ public static IServiceCollection AddMongoStorages(this IServiceCollection servic .AddSingleton(); } - if (components["ObjectStorage"] == "ArmoniK.Adapters.MongoDB.ObjectStorage") - { - services.AddOption(configuration, - Options.ObjectStorage.SettingSection) - .AddSingleton() - .AddSingleton(); - } - services.AddOption(configuration, Options.MongoDB.SettingSection, out var mongoOptions); diff --git a/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs b/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs index 1621b252c..f4ba0cd91 100644 --- a/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs +++ b/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs @@ -53,7 +53,7 @@ public ResultDataModelMapping() cm.MapProperty(nameof(Result.Size)) .SetIgnoreIfDefault(true) .SetDefaultValue(0); - cm.MapProperty(nameof(Result.Data)) + cm.MapProperty(nameof(Result.OpaqueId)) .SetIsRequired(true); cm.SetIgnoreExtraElements(true); cm.MapCreator(model => new Result(model.SessionId, @@ -65,7 +65,7 @@ public ResultDataModelMapping() model.DependentTasks, model.CreationDate, model.Size, - model.Data)); + model.OpaqueId)); }); } } diff --git a/Adaptors/MongoDB/src/TaskTable.cs b/Adaptors/MongoDB/src/TaskTable.cs index 50f2ef468..2f42590f7 100644 --- a/Adaptors/MongoDB/src/TaskTable.cs +++ b/Adaptors/MongoDB/src/TaskTable.cs @@ -27,6 +27,7 @@ using ArmoniK.Core.Adapters.MongoDB.Options; using ArmoniK.Core.Adapters.MongoDB.Table.DataModel; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; @@ -370,11 +371,11 @@ public async Task RemoveRemainingDataDependenciesAsync(ICollection taskI } - var key0 = TaskData.EscapeKey(deps.Current); + var key0 = deps.Current; var update = new UpdateDefinitionBuilder().Unset(data => data.RemainingDataDependencies[key0]); while (deps.MoveNext()) { - var key = TaskData.EscapeKey(deps.Current); + var key = deps.Current; update = update.Unset(data => data.RemainingDataDependencies[key]); } diff --git a/Adaptors/MongoDB/tests/ArmoniK.Core.Adapters.MongoDB.Tests.csproj b/Adaptors/MongoDB/tests/ArmoniK.Core.Adapters.MongoDB.Tests.csproj index 401f9dc8d..5d89e35c1 100644 --- a/Adaptors/MongoDB/tests/ArmoniK.Core.Adapters.MongoDB.Tests.csproj +++ b/Adaptors/MongoDB/tests/ArmoniK.Core.Adapters.MongoDB.Tests.csproj @@ -13,7 +13,7 @@ - + diff --git a/Adaptors/MongoDB/tests/BsonSerializerTest.cs b/Adaptors/MongoDB/tests/BsonSerializerTest.cs index 269da283d..940f64f31 100644 --- a/Adaptors/MongoDB/tests/BsonSerializerTest.cs +++ b/Adaptors/MongoDB/tests/BsonSerializerTest.cs @@ -119,7 +119,7 @@ public void SerializeResultDataModel() deserialized.DependentTasks); Assert.AreEqual(rdm.CreationDate, deserialized.CreationDate); - Assert.IsTrue(rdm.Data.SequenceEqual(deserialized.Data)); + Assert.IsTrue(rdm.OpaqueId.SequenceEqual(deserialized.OpaqueId)); } [Test] diff --git a/Adaptors/MongoDB/tests/InjectionTests.cs b/Adaptors/MongoDB/tests/InjectionTests.cs index ab8fc22b5..ca634d841 100644 --- a/Adaptors/MongoDB/tests/InjectionTests.cs +++ b/Adaptors/MongoDB/tests/InjectionTests.cs @@ -89,9 +89,6 @@ public void SetUp() { $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.TableStorage)}:PollingDelayMax", "00:00:20" }, - { - $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.ObjectStorage)}:ChunkSize", "100000" - }, }; var logger = NullLogger.Instance; @@ -238,23 +235,6 @@ public void ReadTablePollingMaxDelay() options.PollingDelayMax); } - [Test] - public void ObjectOptionsNotNull() - { - var options = provider_!.GetRequiredService(); - - Assert.NotNull(options); - } - - [Test] - public void ReadObjectChunkSize() - { - var options = provider_!.GetRequiredService(); - - Assert.AreEqual(100000, - options.ChunkSize); - } - [Test] public void BuildTableStorage() { @@ -280,22 +260,4 @@ public void TableStorageHasPollingDelayMax() Assert.AreEqual(TimeSpan.FromSeconds(20), table.PollingDelayMax); } - - [Test] - public void BuildObjectStorage() - { - var objectStorage = provider_!.GetRequiredService(); - - Assert.NotNull(objectStorage); - } - - [Test] - public void ObjectStorageFactoryHasBindingToObjectStorage() - { - var objectStorage = provider_!.GetRequiredService(); - - Assert.NotNull(objectStorage); - Assert.AreEqual(typeof(ObjectStorage), - objectStorage.GetType()); - } } diff --git a/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs b/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs index 2ae12b790..0188f697f 100644 --- a/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs +++ b/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs @@ -84,9 +84,6 @@ public MongoDatabaseProvider(bool useSingleNodeReplicaSet { $"{Components.SettingSection}:{nameof(Components.TableStorage)}", "ArmoniK.Adapters.MongoDB.TableStorage" }, - { - $"{Components.SettingSection}:{nameof(Components.ObjectStorage)}", "ArmoniK.Adapters.MongoDB.ObjectStorage" - }, { $"{Components.SettingSection}:{nameof(Components.AuthenticationStorage)}", "ArmoniK.Adapters.MongoDB.AuthenticationTable" @@ -98,10 +95,6 @@ public MongoDatabaseProvider(bool useSingleNodeReplicaSet $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.TableStorage)}:{nameof(Options.MongoDB.TableStorage.PollingDelayMax)}", "00:00:10" }, - { - $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.ObjectStorage)}:{nameof(Options.MongoDB.ObjectStorage.ChunkSize)}", - "140000" - }, }; var configuration = new ConfigurationManager(); diff --git a/Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj b/Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj index 2a179f55d..861f55507 100644 --- a/Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj +++ b/Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj @@ -22,19 +22,19 @@ runtime - + runtime runtime - + runtime - + runtime - + runtime diff --git a/Adaptors/QueueCommon/src/ArmoniK.Core.Adapters.QueueCommon.csproj b/Adaptors/QueueCommon/src/ArmoniK.Core.Adapters.QueueCommon.csproj index 15402e11c..4b3fe3ef0 100644 --- a/Adaptors/QueueCommon/src/ArmoniK.Core.Adapters.QueueCommon.csproj +++ b/Adaptors/QueueCommon/src/ArmoniK.Core.Adapters.QueueCommon.csproj @@ -29,7 +29,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + false runtime diff --git a/Adaptors/RabbitMQ/src/ArmoniK.Core.Adapters.RabbitMQ.csproj b/Adaptors/RabbitMQ/src/ArmoniK.Core.Adapters.RabbitMQ.csproj index 33400b9c3..1bde22949 100644 --- a/Adaptors/RabbitMQ/src/ArmoniK.Core.Adapters.RabbitMQ.csproj +++ b/Adaptors/RabbitMQ/src/ArmoniK.Core.Adapters.RabbitMQ.csproj @@ -31,7 +31,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + false runtime diff --git a/Adaptors/RabbitMQ/src/ConnectionRabbit.cs b/Adaptors/RabbitMQ/src/ConnectionRabbit.cs index 57924b3e6..e1f9542e3 100644 --- a/Adaptors/RabbitMQ/src/ConnectionRabbit.cs +++ b/Adaptors/RabbitMQ/src/ConnectionRabbit.cs @@ -37,35 +37,37 @@ namespace ArmoniK.Core.Adapters.RabbitMQ; [UsedImplicitly] public class ConnectionRabbit : IConnectionRabbit { - private readonly AsyncLazy connectionTask_; - private readonly ILogger logger_; - - private readonly Amqp options_; - - private bool isInitialized_; + private readonly ExecutionSingleizer connectionSingleizer_ = new(); + private readonly ConnectionFactory factory_; + private readonly ILogger logger_; + private readonly Amqp options_; + private IModel? model_; public ConnectionRabbit(Amqp options, ILogger logger) { - logger_ = logger; - options_ = options; - connectionTask_ = new AsyncLazy(() => InitTask(this)); + logger_ = logger; + options_ = options; + factory_ = new ConnectionFactory + { + UserName = options.User, + Password = options.Password, + HostName = options.Host, + Port = options.Port, + DispatchConsumersAsync = true, + }; } - private IConnection? Connection { get; set; } - - public IModel? Channel { get; private set; } - - public async Task Init(CancellationToken cancellationToken = default) - => await connectionTask_; + public Task Init(CancellationToken cancellationToken = default) + => GetConnectionAsync(cancellationToken); public Task Check(HealthCheckTag tag) => tag switch { - HealthCheckTag.Startup or HealthCheckTag.Readiness => Task.FromResult(isInitialized_ + HealthCheckTag.Startup or HealthCheckTag.Readiness => Task.FromResult(model_ is not null ? HealthCheckResult.Healthy() : HealthCheckResult.Unhealthy($"{nameof(ConnectionRabbit)} is not yet initialized.")), - HealthCheckTag.Liveness => Task.FromResult(isInitialized_ && Connection is not null && Connection.IsOpen && Channel is not null && Channel.IsOpen + HealthCheckTag.Liveness => Task.FromResult(model_ is not null && model_.IsOpen ? HealthCheckResult.Healthy() : HealthCheckResult.Unhealthy($"{nameof(ConnectionRabbit)} not initialized or connection dropped.")), _ => throw new ArgumentOutOfRangeException(nameof(tag), @@ -75,35 +77,51 @@ public Task Check(HealthCheckTag tag) public void Dispose() { - if (isInitialized_) - { - Channel!.Close(); - Connection!.Close(); + model_?.Close(); + model_?.Dispose(); - Channel.Dispose(); - Connection.Dispose(); - } + connectionSingleizer_.Dispose(); GC.SuppressFinalize(this); } - private async Task InitTask(ConnectionRabbit conn, - CancellationToken cancellationToken = default) + + public async Task GetConnectionAsync(CancellationToken cancellationToken = default) { - var factory = new ConnectionFactory - { - UserName = conn.options_.User, - Password = conn.options_.Password, - HostName = conn.options_.Host, - Port = conn.options_.Port, - DispatchConsumersAsync = true, - }; + if (model_ is not null && !model_.IsClosed) + { + return model_; + } + return await connectionSingleizer_.Call(async token => + { + // this is needed to resolve TOCTOU problem + if (model_ is not null && !model_.IsClosed) + { + return model_; + } + + var conn = await CreateConnection(options_, + factory_, + logger_, + token) + .ConfigureAwait(false); + model_ = conn; + return conn; + }, + cancellationToken) + .ConfigureAwait(false); + } - if (options_.Scheme.Equals("AMQPS")) + private static async Task CreateConnection(Amqp options, + ConnectionFactory factory, + ILogger logger, + CancellationToken cancellationToken = default) + { + if (options.Scheme.Equals("AMQPS")) { factory.Ssl.Enabled = true; - factory.Ssl.ServerName = conn.options_.Host; + factory.Ssl.ServerName = options.Host; factory.Ssl.CertificateValidationCallback = delegate(object _, X509Certificate? _, X509Chain? _, @@ -111,51 +129,46 @@ private async Task InitTask(ConnectionRabbit conn, { switch (errors) { - case SslPolicyErrors.RemoteCertificateNameMismatch when conn.options_.AllowHostMismatch: + case SslPolicyErrors.RemoteCertificateNameMismatch when options.AllowHostMismatch: case SslPolicyErrors.None: return true; default: - logger_.LogError("SSL error : {error}", - errors); + logger.LogError("SSL error : {error}", + errors); return false; } }; } var retry = 0; - for (; retry < conn.options_.MaxRetries; retry++) + for (; retry < options.MaxRetries; retry++) { try { - conn.Connection = factory.CreateConnection(); - conn.Connection.ConnectionShutdown += (_, - ea) => OnShutDown(ea, - "Connection", - logger_); - - Channel = conn.Connection.CreateModel(); - Channel.ModelShutdown += (_, - ea) => OnShutDown(ea, - "Channel", - logger_); - break; + var connection = factory.CreateConnection(); + connection.ConnectionShutdown += (_, + ea) => OnShutDown(ea, + "Connection", + logger); + + var model = connection.CreateModel(); + model.ModelShutdown += (_, + ea) => OnShutDown(ea, + "Channel", + logger); + return model; } catch (Exception ex) { - logger_.LogInformation(ex, - "Retrying to create connection"); + logger.LogInformation(ex, + "Retrying to create connection"); await Task.Delay(1000 * retry, cancellationToken) .ConfigureAwait(false); } } - if (retry == conn.options_.MaxRetries) - { - throw new TimeoutException($"{nameof(conn.options_.MaxRetries)} reached"); - } - - conn.isInitialized_ = true; + throw new TimeoutException($"{nameof(options.MaxRetries)} reached"); } private static void OnShutDown(ShutdownEventArgs ea, diff --git a/Adaptors/RabbitMQ/src/IConnectionRabbit.cs b/Adaptors/RabbitMQ/src/IConnectionRabbit.cs index 7b50e4426..0116fe92e 100644 --- a/Adaptors/RabbitMQ/src/IConnectionRabbit.cs +++ b/Adaptors/RabbitMQ/src/IConnectionRabbit.cs @@ -16,6 +16,8 @@ // along with this program. If not, see . using System; +using System.Threading; +using System.Threading.Tasks; using ArmoniK.Core.Base; @@ -25,5 +27,5 @@ namespace ArmoniK.Core.Adapters.RabbitMQ; public interface IConnectionRabbit : IInitializable, IDisposable { - public IModel? Channel { get; } + Task GetConnectionAsync(CancellationToken cancellationToken = default); } diff --git a/Adaptors/RabbitMQ/src/PullQueueStorage.cs b/Adaptors/RabbitMQ/src/PullQueueStorage.cs index ed5e56d3e..b4d6247df 100644 --- a/Adaptors/RabbitMQ/src/PullQueueStorage.cs +++ b/Adaptors/RabbitMQ/src/PullQueueStorage.cs @@ -69,11 +69,15 @@ await ConnectionRabbit.Init(cancellationToken) }, }; - ConnectionRabbit.Channel!.QueueDeclare(Options!.PartitionId, - false, /* to survive broker restart */ - false, /* used by multiple connections */ - false, /* not deleted when last consumer unsubscribes (if it has had one) */ - queueArgs); + var connection = await ConnectionRabbit.GetConnectionAsync(cancellationToken) + .ConfigureAwait(false); + + connection.QueueDeclare(Options!.PartitionId, + false, /* to survive broker restart */ + false, /* used by multiple connections */ + false, /* not deleted when last consumer unsubscribes (if it has had one) */ + queueArgs); + IsInitialized = true; } @@ -91,8 +95,11 @@ public async IAsyncEnumerable PullMessagesAsync(int { cancellationToken.ThrowIfCancellationRequested(); - var message = ConnectionRabbit.Channel!.BasicGet(Options.PartitionId, - false); + var connection = await ConnectionRabbit.GetConnectionAsync(cancellationToken) + .ConfigureAwait(false); + + var message = connection.BasicGet(Options.PartitionId, + false); if (message is null) { @@ -103,7 +110,7 @@ await Task.Delay(Delay, } nbPulledMessage++; - yield return new QueueMessageHandler(ConnectionRabbit.Channel!, + yield return new QueueMessageHandler(ConnectionRabbit, message, Encoding.UTF8.GetString(message.Body.ToArray()), cancellationToken); diff --git a/Adaptors/RabbitMQ/src/PushQueueStorage.cs b/Adaptors/RabbitMQ/src/PushQueueStorage.cs index ca20a8eef..f3ad0150f 100644 --- a/Adaptors/RabbitMQ/src/PushQueueStorage.cs +++ b/Adaptors/RabbitMQ/src/PushQueueStorage.cs @@ -63,14 +63,17 @@ private Task PushMessagesAsync(IEnumerable messages, { var task = Task.Run(() => PushMessages(messages, partitionId, - priority), + priority, + cancellationToken), cancellationToken); return task; } - private void PushMessages(IEnumerable messages, - string partitionId, - int priority) + private async Task PushMessages(IEnumerable messages, + string partitionId, + int priority, + CancellationToken cancellationToken) + { if (!IsInitialized) { @@ -80,29 +83,34 @@ private void PushMessages(IEnumerable messages, var queueArgs = new Dictionary { { - "x-max-priority", Options!.MaxPriority + "x-max-priority", Options.MaxPriority }, { "x-queue-mode", "lazy" // queue will try to move messages to disk as early as practically possible }, }; - ConnectionRabbit.Channel!.QueueDeclare(partitionId, - false, /* to survive broker restart */ - false, /* used by multiple connections */ - false, /* not deleted when last consumer unsubscribes (if it has had one) */ - queueArgs); + var connection = await ConnectionRabbit.GetConnectionAsync(CancellationToken.None) + .ConfigureAwait(false); + + connection.QueueDeclare(partitionId, + false, /* to survive broker restart */ + false, /* used by multiple connections */ + false, /* not deleted when last consumer unsubscribes (if it has had one) */ + queueArgs); foreach (var msg in messages) { - var basicProperties = ConnectionRabbit.Channel!.CreateBasicProperties(); + connection = await ConnectionRabbit.GetConnectionAsync(CancellationToken.None) + .ConfigureAwait(false); + var basicProperties = connection.CreateBasicProperties(); basicProperties.Priority = Convert.ToByte(priority); basicProperties.MessageId = Guid.NewGuid() .ToString(); - ConnectionRabbit.Channel.BasicPublish("", - partitionId, - basicProperties, - Encoding.UTF8.GetBytes(msg.TaskId)); + connection.BasicPublish("", + partitionId, + basicProperties, + Encoding.UTF8.GetBytes(msg.TaskId)); } } } diff --git a/Adaptors/RabbitMQ/src/QueueMessageHandler.cs b/Adaptors/RabbitMQ/src/QueueMessageHandler.cs index 3569e189b..31b0950cd 100644 --- a/Adaptors/RabbitMQ/src/QueueMessageHandler.cs +++ b/Adaptors/RabbitMQ/src/QueueMessageHandler.cs @@ -27,10 +27,10 @@ namespace ArmoniK.Core.Adapters.RabbitMQ; public class QueueMessageHandler : IQueueMessageHandler { - private readonly BasicGetResult basicGetResult_; - private readonly IModel channel_; + private readonly BasicGetResult basicGetResult_; + private readonly IConnectionRabbit channel_; - public QueueMessageHandler(IModel channel, + public QueueMessageHandler(IConnectionRabbit channel, BasicGetResult basicGetResult, string taskId, CancellationToken cancellationToken) @@ -58,8 +58,11 @@ public string MessageId /// public DateTime ReceptionDateTime { get; init; } - public ValueTask DisposeAsync() + public async ValueTask DisposeAsync() { + var connection = await channel_.GetConnectionAsync(CancellationToken.None) + .ConfigureAwait(false); + switch (Status) { case QueueMessageStatus.Postponed: @@ -68,22 +71,22 @@ public ValueTask DisposeAsync() case QueueMessageStatus.Waiting: /* Negative acknowledging this message will send it to the retry exchange, see PullQueueStorage.cs */ - channel_.BasicNack(basicGetResult_.DeliveryTag, - false, - true); + connection.BasicNack(basicGetResult_.DeliveryTag, + false, + true); break; case QueueMessageStatus.Cancelled: case QueueMessageStatus.Processed: - channel_.BasicAck(basicGetResult_.DeliveryTag, - false); + connection.BasicAck(basicGetResult_.DeliveryTag, + false); break; case QueueMessageStatus.Poisonous: - channel_.BasicNack(basicGetResult_.DeliveryTag, - false, - false); + connection.BasicNack(basicGetResult_.DeliveryTag, + false, + false); break; default: @@ -93,7 +96,5 @@ public ValueTask DisposeAsync() } GC.SuppressFinalize(this); - - return ValueTask.CompletedTask; } } diff --git a/Adaptors/RabbitMQ/tests/ArmoniK.Core.Adapters.RabbitMQ.Tests.csproj b/Adaptors/RabbitMQ/tests/ArmoniK.Core.Adapters.RabbitMQ.Tests.csproj index eee4ffb3c..7ec09cb9e 100644 --- a/Adaptors/RabbitMQ/tests/ArmoniK.Core.Adapters.RabbitMQ.Tests.csproj +++ b/Adaptors/RabbitMQ/tests/ArmoniK.Core.Adapters.RabbitMQ.Tests.csproj @@ -20,7 +20,7 @@ - + diff --git a/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj b/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj index c33e36666..da557fc65 100644 --- a/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj +++ b/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj @@ -1,4 +1,4 @@ - + net8.0 win-x64;linux-x64;linux-arm64 @@ -9,18 +9,67 @@ True true enable + true + + Embedded + true + DEBUG;TRACE + + + + true + true + snupkg + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + runtime + + + runtime + + + runtime + + + false + runtime + + + runtime + + + runtime + all runtime; build; native; contentfiles; analyzers; buildtransitive - + + runtime + - + + false + runtime + + + false + runtime + diff --git a/Adaptors/Redis/src/ObjectBuilder.cs b/Adaptors/Redis/src/ObjectBuilder.cs new file mode 100644 index 000000000..0374b48c9 --- /dev/null +++ b/Adaptors/Redis/src/ObjectBuilder.cs @@ -0,0 +1,117 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.IO; +using System.Security.Cryptography.X509Certificates; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Utils; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using StackExchange.Redis; + +namespace ArmoniK.Core.Adapters.Redis; + +/// +/// Class for building Redis instance and Object interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable +{ + /// + [PublicAPI] + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) + { + // ReSharper disable once InlineOutVariableDeclaration + Options.Redis redisOptions; + serviceCollection.AddOption(configuration, + Options.Redis.SettingSection, + out redisOptions); + + if (!string.IsNullOrEmpty(redisOptions.CredentialsPath)) + { + configuration.AddJsonFile(redisOptions.CredentialsPath, + false, + false); + + serviceCollection.AddOption(configuration, + Options.Redis.SettingSection, + out redisOptions); + + logger.LogTrace("Loaded Redis credentials from file {path}", + redisOptions.CredentialsPath); + } + + if (!string.IsNullOrEmpty(redisOptions.CaPath)) + { + var localTrustStore = new X509Store(StoreName.Root); + var certificateCollection = new X509Certificate2Collection(); + try + { + certificateCollection.ImportFromPemFile(redisOptions.CaPath); + localTrustStore.Open(OpenFlags.ReadWrite); + localTrustStore.AddRange(certificateCollection); + logger.LogTrace("Imported Redis certificate from file {path}", + redisOptions.CaPath); + } + catch (Exception ex) + { + logger.LogError("Root certificate import failed: {error}", + ex.Message); + throw; + } + finally + { + localTrustStore.Close(); + } + } + + var config = new ConfigurationOptions + { + ClientName = redisOptions.ClientName, + ReconnectRetryPolicy = new ExponentialRetry(10), + Ssl = redisOptions.Ssl, + AbortOnConnectFail = true, + SslHost = redisOptions.SslHost, + Password = redisOptions.Password, + User = redisOptions.User, + }; + config.EndPoints.Add(redisOptions.EndpointUrl); + + if (redisOptions.Timeout > 0) + { + config.ConnectTimeout = redisOptions.Timeout; + } + + logger.LogDebug("setup connection to Redis at {EndpointUrl} with user {user}", + redisOptions.EndpointUrl, + redisOptions.User); + + serviceCollection.AddSingleton(_ => ConnectionMultiplexer.Connect(config, + TextWriter.Null) + .GetDatabase()); + serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); + } +} diff --git a/Adaptors/Redis/src/ObjectStorage.cs b/Adaptors/Redis/src/ObjectStorage.cs index 9acf36ee0..bb45932c1 100644 --- a/Adaptors/Redis/src/ObjectStorage.cs +++ b/Adaptors/Redis/src/ObjectStorage.cs @@ -19,13 +19,13 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; -using ArmoniK.Api.Common.Utils; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Utils; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -87,13 +87,14 @@ public Task Check(HealthCheckTag tag) }; /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { - long size = 0; - var storageNameKey = objectStorageName_ + key; - using var _ = logger_.LogFunction(storageNameKey); + var key = Guid.NewGuid() + .ToString(); + var storageNameKey = objectStorageName_ + key; + long size = 0; var idx = 0; var taskList = new List(); @@ -112,14 +113,14 @@ public async Task AddOrUpdateAsync(string await taskList.WhenAll() .ConfigureAwait(false); - return size; + return (Encoding.UTF8.GetBytes(key), size); } /// - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - using var _ = logger_.LogFunction(objectStorageName_ + key); + var key = Encoding.UTF8.GetString(id); var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count")) .ConfigureAwait(false); @@ -145,20 +146,17 @@ public async IAsyncEnumerable GetValuesAsync(string } /// - public async Task TryDeleteAsync(IEnumerable keys, + public async Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) - => await keys.ParallelForEach(key => TryDeleteAsync(key, - cancellationToken)) - .ConfigureAwait(false); - - /// - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); + => await ids.ParallelForEach(id => TryDeleteAsync(id, + cancellationToken)) + .ConfigureAwait(false); - private async Task TryDeleteAsync(string key, + private async Task TryDeleteAsync(byte[] id, CancellationToken cancellationToken = default) { - using var _ = logger_.LogFunction(objectStorageName_ + key); + var key = Encoding.UTF8.GetString(id); + var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count")) .ConfigureAwait(false); diff --git a/Adaptors/Redis/src/ServiceCollectionExt.cs b/Adaptors/Redis/src/ServiceCollectionExt.cs deleted file mode 100644 index 79c8ed094..000000000 --- a/Adaptors/Redis/src/ServiceCollectionExt.cs +++ /dev/null @@ -1,124 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.IO; -using System.Security.Cryptography.X509Certificates; - -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Common.Injection.Options; -using ArmoniK.Core.Common.Storage; -using ArmoniK.Core.Utils; - -using JetBrains.Annotations; - -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -using StackExchange.Redis; - -namespace ArmoniK.Core.Adapters.Redis; - -public static class ServiceCollectionExt -{ - [PublicAPI] - public static IServiceCollection AddRedis(this IServiceCollection serviceCollection, - ConfigurationManager configuration, - ILogger logger) - { - var components = configuration.GetSection(Components.SettingSection); - - if (components["ObjectStorage"] == "ArmoniK.Adapters.Redis.ObjectStorage") - { - // ReSharper disable once InlineOutVariableDeclaration - Options.Redis redisOptions; - serviceCollection.AddOption(configuration, - Options.Redis.SettingSection, - out redisOptions); - - using var _ = logger.BeginNamedScope("Redis configuration", - ("EndpointUrl", redisOptions.EndpointUrl)); - - if (!string.IsNullOrEmpty(redisOptions.CredentialsPath)) - { - configuration.AddJsonFile(redisOptions.CredentialsPath, - false, - false); - - serviceCollection.AddOption(configuration, - Options.Redis.SettingSection, - out redisOptions); - - logger.LogTrace("Loaded Redis credentials from file {path}", - redisOptions.CredentialsPath); - } - - if (!string.IsNullOrEmpty(redisOptions.CaPath)) - { - var localTrustStore = new X509Store(StoreName.Root); - var certificateCollection = new X509Certificate2Collection(); - try - { - certificateCollection.ImportFromPemFile(redisOptions.CaPath); - localTrustStore.Open(OpenFlags.ReadWrite); - localTrustStore.AddRange(certificateCollection); - logger.LogTrace("Imported Redis certificate from file {path}", - redisOptions.CaPath); - } - catch (Exception ex) - { - logger.LogError("Root certificate import failed: {error}", - ex.Message); - throw; - } - finally - { - localTrustStore.Close(); - } - } - - var config = new ConfigurationOptions - { - ClientName = redisOptions.ClientName, - ReconnectRetryPolicy = new ExponentialRetry(10), - Ssl = redisOptions.Ssl, - AbortOnConnectFail = true, - SslHost = redisOptions.SslHost, - Password = redisOptions.Password, - User = redisOptions.User, - }; - config.EndPoints.Add(redisOptions.EndpointUrl); - - if (redisOptions.Timeout > 0) - { - config.ConnectTimeout = redisOptions.Timeout; - } - - logger.LogDebug("setup connection to Redis at {EndpointUrl} with user {user}", - redisOptions.EndpointUrl, - redisOptions.User); - - serviceCollection.AddSingleton(_ => ConnectionMultiplexer.Connect(config, - TextWriter.Null) - .GetDatabase()); - serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); - } - - return serviceCollection; - } -} diff --git a/Adaptors/Redis/tests/ArmoniK.Core.Adapters.Redis.Tests.csproj b/Adaptors/Redis/tests/ArmoniK.Core.Adapters.Redis.Tests.csproj index 1d78800d1..0ba3a5a84 100644 --- a/Adaptors/Redis/tests/ArmoniK.Core.Adapters.Redis.Tests.csproj +++ b/Adaptors/Redis/tests/ArmoniK.Core.Adapters.Redis.Tests.csproj @@ -13,7 +13,7 @@ - + diff --git a/Adaptors/Redis/tests/ObjectStorageTests.cs b/Adaptors/Redis/tests/ObjectStorageTests.cs index a133b4753..cd43ef46f 100644 --- a/Adaptors/Redis/tests/ObjectStorageTests.cs +++ b/Adaptors/Redis/tests/ObjectStorageTests.cs @@ -19,7 +19,7 @@ using System.Collections.Generic; using System.IO; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base; using ArmoniK.Core.Common.Tests.TestBase; using ArmoniK.Core.Utils; @@ -53,9 +53,6 @@ protected override void GetObjectStorageInstance() // Minimal set of configurations to operate on a toy DB Dictionary minimalConfig = new() { - { - "Components:ObjectStorage", "ArmoniK.Adapters.Redis.ObjectStorage" - }, { "Redis:MaxRetry", "5" }, diff --git a/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj b/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj index d78e792d1..d2064ca6e 100644 --- a/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj +++ b/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj @@ -2,8 +2,14 @@ net8.0 win-x64;linux-x64;linux-arm64 - enable + ANEO + Copyright (C) ANEO, 2021-2021 + AGPL-3.0-or-later + True + True + true enable + true @@ -11,12 +17,50 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + + + - + + runtime + + + runtime + + + runtime + + + false + runtime + + + runtime + + + runtime + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + runtime + + + + + + false + runtime + + + false + runtime + diff --git a/Adaptors/S3/src/ObjectBuilder.cs b/Adaptors/S3/src/ObjectBuilder.cs new file mode 100644 index 000000000..599a64846 --- /dev/null +++ b/Adaptors/S3/src/ObjectBuilder.cs @@ -0,0 +1,77 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using Amazon.S3; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Utils; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace ArmoniK.Core.Adapters.S3; + +/// +/// Class for building S3 instance and Object interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable +{ + /// + [PublicAPI] + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) + { + // ReSharper disable once InlineOutVariableDeclaration + Options.S3 s3Options; + serviceCollection.AddOption(configuration, + Options.S3.SettingSection, + out s3Options); + + logger.LogInformation("setup connection to S3 at {EndpointUrl} with user {user} with option ForcePathStyle = {ForcePathStyle} with BucketName = {BucketName}", + s3Options.EndpointUrl, + s3Options.Login, + s3Options.MustForcePathStyle, + s3Options.BucketName); + + var s3Config = new AmazonS3Config + { + ForcePathStyle = s3Options.MustForcePathStyle, + ServiceURL = s3Options.EndpointUrl, + }; + + AmazonS3Client s3Client; + if (string.IsNullOrWhiteSpace(s3Options.Login)) + { + s3Client = new AmazonS3Client(s3Config); + } + else + { + s3Client = new AmazonS3Client(s3Options.Login, + s3Options.Password, + s3Config); + } + + + serviceCollection.AddSingleton(_ => s3Client); + serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); + } +} diff --git a/Adaptors/S3/src/ObjectStorage.cs b/Adaptors/S3/src/ObjectStorage.cs index d8bb7bf0a..d63acb17a 100644 --- a/Adaptors/S3/src/ObjectStorage.cs +++ b/Adaptors/S3/src/ObjectStorage.cs @@ -15,16 +15,22 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; using Amazon.S3; using Amazon.S3.Model; using Amazon.S3.Util; -using ArmoniK.Api.Common.Utils; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Utils; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -90,13 +96,12 @@ public Task Check(HealthCheckTag tag) }; /// - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var objectStorageFullName = $"{objectStorageName_}{key}"; - using var loggerFunction = logger_.LogFunction(objectStorageFullName); - using var loggerContext = logger_.BeginPropertyScope(("objectKey", key), - ("@S3Options", options_.Confidential())); + var key = Encoding.UTF8.GetString(id); + var objectStorageFullName = $"{objectStorageName_}{key}"; + try { await s3Client_.GetObjectAsync(options_.BucketName, @@ -160,30 +165,19 @@ await s3Client_.GetObjectAsync(options_.BucketName, } /// - public async Task TryDeleteAsync(IEnumerable keys, - CancellationToken cancellationToken = default) - => await keys.ParallelForEach(key => TryDeleteAsync(key, - cancellationToken)) - .ConfigureAwait(false); - - /// - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); - - /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { long[] sizeBox = { 0, }; - var objectStorageFullName = $"{objectStorageName_}{key}"; - using var loggerFunction = logger_.LogFunction(objectStorageFullName); - using var loggerContext = logger_.BeginPropertyScope(("objectKey", key), - ("@S3Options", options_.Confidential())); + var key = Guid.NewGuid() + .ToString(); + var objectStorageFullName = $"{objectStorageName_}{key}"; + logger_.LogDebug("Upload object"); var initRequest = new InitiateMultipartUploadRequest { @@ -234,16 +228,22 @@ await s3Client_.AbortMultipartUploadAsync(abortMpuRequest, throw; } - return sizeBox[0]; + return (Encoding.UTF8.GetBytes(key), sizeBox[0]); } - private async Task TryDeleteAsync(string key, + /// + public async Task TryDeleteAsync(IEnumerable ids, + CancellationToken cancellationToken = default) + => await ids.ParallelForEach(id => TryDeleteAsync(id, + cancellationToken)) + .ConfigureAwait(false); + + private async Task TryDeleteAsync(byte[] id, CancellationToken cancellationToken = default) { - var objectStorageFullName = $"{objectStorageName_}{key}"; - using var loggerFunction = logger_.LogFunction(objectStorageFullName); - using var loggerContext = logger_.BeginPropertyScope(("objectKey", key), - ("@S3Options", options_.Confidential())); + var key = Encoding.UTF8.GetString(id); + var objectStorageFullName = $"{objectStorageName_}{key}"; + var objectDeleteRequest = new DeleteObjectRequest { BucketName = options_.BucketName, diff --git a/Adaptors/S3/src/ServiceCollectionExt.cs b/Adaptors/S3/src/ServiceCollectionExt.cs deleted file mode 100644 index 97d415ba8..000000000 --- a/Adaptors/S3/src/ServiceCollectionExt.cs +++ /dev/null @@ -1,84 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using Amazon.S3; - -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Common.Injection.Options; -using ArmoniK.Core.Common.Storage; -using ArmoniK.Core.Utils; - -using JetBrains.Annotations; - -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace ArmoniK.Core.Adapters.S3; - -public static class ServiceCollectionExt -{ - [PublicAPI] - public static IServiceCollection AddS3(this IServiceCollection serviceCollection, - ConfigurationManager configuration, - ILogger logger) - { - var components = configuration.GetSection(Components.SettingSection); - - if (components["ObjectStorage"] == "ArmoniK.Adapters.S3.ObjectStorage") - { - // ReSharper disable once InlineOutVariableDeclaration - Options.S3 s3Options; - serviceCollection.AddOption(configuration, - Options.S3.SettingSection, - out s3Options); - - using var _ = logger.BeginNamedScope("S3 configuration", - ("EndpointUrl", s3Options.EndpointUrl)); - - logger.LogInformation("setup connection to S3 at {EndpointUrl} with user {user} with option ForcePathStyle = {ForcePathStyle} with BucketName = {BucketName}", - s3Options.EndpointUrl, - s3Options.Login, - s3Options.MustForcePathStyle, - s3Options.BucketName); - - var s3Config = new AmazonS3Config - { - ForcePathStyle = s3Options.MustForcePathStyle, - ServiceURL = s3Options.EndpointUrl, - }; - - AmazonS3Client s3Client; - if (string.IsNullOrWhiteSpace(s3Options.Login)) - { - s3Client = new AmazonS3Client(s3Config); - } - else - { - s3Client = new AmazonS3Client(s3Options.Login, - s3Options.Password, - s3Config); - } - - - serviceCollection.AddSingleton(_ => s3Client); - serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); - } - - return serviceCollection; - } -} diff --git a/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj b/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj index abd2a01a3..ae9d8bc16 100644 --- a/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj +++ b/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj @@ -12,7 +12,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + @@ -27,7 +27,6 @@ - diff --git a/Adaptors/S3/tests/ObjectStorageTests.cs b/Adaptors/S3/tests/ObjectStorageTests.cs index 05996f044..db6615666 100644 --- a/Adaptors/S3/tests/ObjectStorageTests.cs +++ b/Adaptors/S3/tests/ObjectStorageTests.cs @@ -15,7 +15,9 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base; +using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Tests.TestBase; using ArmoniK.Core.Common.Utils; @@ -35,12 +37,25 @@ public override void TearDown() RunTests = false; } + private static readonly string SolutionRoot = + Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(typeof(ObjectStorageTests) + .Assembly + .Location))))) ?? + string.Empty)); + + private static readonly string S3Path = + $"{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.S3.dll"; + + protected override void GetObjectStorageInstance() { Dictionary minimalConfig = new() { { - "Components:ObjectStorage", "ArmoniK.Adapters.S3.ObjectStorage" + "Components:ObjectStorageAdaptorSettings:ClassName", "ArmoniK.Core.Adapters.S3.ObjectBuilder" + }, + { + "Components:ObjectStorageAdaptorSettings:AdapterAbsolutePath", $"{SolutionRoot}{S3Path}" }, { "S3:BucketName", "miniobucket" @@ -66,10 +81,9 @@ protected override void GetObjectStorageInstance() services.AddLogging(); var logger = new LoggerInit(configuration); - services.AddS3(configuration, - logger.GetLogger()); - - services.AddSingleton(); + services.AddAdapter(configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()); var provider = services.BuildServiceProvider(new ServiceProviderOptions { diff --git a/Adaptors/SQS/src/AmazonSQSClient.cs b/Adaptors/SQS/src/AmazonSQSClient.cs index 7e63a4dfa..91b66e7f8 100644 --- a/Adaptors/SQS/src/AmazonSQSClient.cs +++ b/Adaptors/SQS/src/AmazonSQSClient.cs @@ -15,6 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -25,9 +26,10 @@ namespace ArmoniK.Core.Adapters.SQS; internal static class AmazonSqsClientExt { - public static async Task GetOrCreateQueueUrlAsync(this AmazonSQSClient client, - string queueName, - CancellationToken cancellationToken) + public static async Task GetOrCreateQueueUrlAsync(this AmazonSQSClient client, + string queueName, + Dictionary tags, + CancellationToken cancellationToken) { try { @@ -40,9 +42,26 @@ public static async Task GetOrCreateQueueUrlAsync(this AmazonSQSClient c return (await client.CreateQueueAsync(new CreateQueueRequest { QueueName = queueName, + Tags = tags, }, cancellationToken) .ConfigureAwait(false)).QueueUrl; } } + + public static string GetQueueName(this AmazonSQSClient client, + SQS options, + string? partition = null) + { + _ = client; + + if (string.IsNullOrEmpty(partition)) + { + partition = options.PartitionId; + } + + return string.IsNullOrEmpty(options.Prefix) + ? partition + : $"{options.Prefix}-{partition}"; + } } diff --git a/Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj b/Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj index 343fab02c..a57c9942b 100644 --- a/Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj +++ b/Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj @@ -8,19 +8,19 @@ - + runtime runtime - + runtime - + runtime - + runtime @@ -33,8 +33,11 @@ - - + + + + + diff --git a/Adaptors/SQS/src/PullQueueStorage.cs b/Adaptors/SQS/src/PullQueueStorage.cs index 5384b95cb..b262eb73d 100644 --- a/Adaptors/SQS/src/PullQueueStorage.cs +++ b/Adaptors/SQS/src/PullQueueStorage.cs @@ -41,9 +41,10 @@ internal class PullQueueStorage : IPullQueueStorage // ReSharper disable once NotAccessedField.Local private readonly ILogger logger_; - private readonly string queueName_; - private bool isInitialized_; - private string? queueUrl_; + private readonly string queueName_; + private readonly Dictionary tags_; + private bool isInitialized_; + private string? queueUrl_; public PullQueueStorage(AmazonSQSClient client, SQS options, @@ -51,7 +52,8 @@ public PullQueueStorage(AmazonSQSClient client, { client_ = client; logger_ = logger; - queueName_ = $"a{options.Prefix}-{options.PartitionId}"; + queueName_ = client.GetQueueName(options); + tags_ = options.Tags; ackDeadlinePeriod_ = options.AckDeadlinePeriod; ackExtendDeadlineStep_ = options.AckExtendDeadlineStep; @@ -95,6 +97,7 @@ public async Task Init(CancellationToken cancellationToken) if (!isInitialized_) { queueUrl_ = await client_.GetOrCreateQueueUrlAsync(queueName_, + tags_, cancellationToken) .ConfigureAwait(false); diff --git a/Adaptors/SQS/src/PushQueueStorage.cs b/Adaptors/SQS/src/PushQueueStorage.cs index bb601627c..49aa10851 100644 --- a/Adaptors/SQS/src/PushQueueStorage.cs +++ b/Adaptors/SQS/src/PushQueueStorage.cs @@ -62,10 +62,12 @@ public async Task PushMessagesAsync(IEnumerable messages, throw new InvalidOperationException($"{nameof(PushQueueStorage)} should be initialized before calling this method."); } - var queueName = $"a{options_.Prefix}-{partitionId}"; + var queueName = client_.GetQueueName(options_, + partitionId); var queueUrl = await cache_.GetOrCreateAsync(queueName, _ => client_.GetOrCreateQueueUrlAsync(queueName, + options_.Tags, cancellationToken)) .ConfigureAwait(false); diff --git a/Adaptors/SQS/src/QueueBuilder.cs b/Adaptors/SQS/src/QueueBuilder.cs index 6c35bc964..c3f090edf 100644 --- a/Adaptors/SQS/src/QueueBuilder.cs +++ b/Adaptors/SQS/src/QueueBuilder.cs @@ -17,7 +17,6 @@ using System; -using Amazon.Runtime; using Amazon.SQS; using ArmoniK.Core.Base; @@ -40,9 +39,7 @@ public void Build(IServiceCollection serviceCollection, var sqsOptions = configuration.GetSection(SQS.SettingSection) .Get() ?? throw new InvalidOperationException("Options not found"); - var credentials = new EnvironmentVariablesAWSCredentials(); - var client = new AmazonSQSClient(credentials, - new AmazonSQSConfig + var client = new AmazonSQSClient(new AmazonSQSConfig { ServiceURL = sqsOptions.ServiceURL, }); diff --git a/Adaptors/SQS/src/SQS.cs b/Adaptors/SQS/src/SQS.cs index 15cf60f38..b36f76afe 100644 --- a/Adaptors/SQS/src/SQS.cs +++ b/Adaptors/SQS/src/SQS.cs @@ -15,6 +15,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using System.Collections.Generic; + namespace ArmoniK.Core.Adapters.SQS; internal class SQS @@ -37,6 +39,11 @@ internal class SQS /// public string Prefix { get; set; } = string.Empty; + /// + /// AWS Tags to add to the Queues when they are created + /// + public Dictionary Tags { get; set; } = new(); + /// /// Acknowledgment deadline in seconds: If a message wasn't acknowledged within this deadline, it will be /// redelivered . diff --git a/ArmoniK.Core.sln b/ArmoniK.Core.sln index 05b8c8138..fb1b1af96 100644 --- a/ArmoniK.Core.sln +++ b/ArmoniK.Core.sln @@ -101,6 +101,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.Core.Tests.Connecti EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.Core.Adapters.SQS", "Adaptors\SQS\src\ArmoniK.Core.Adapters.SQS.csproj", "{399C779C-CE8D-4757-8098-7F055467D96C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArmoniK.Core.Adapters.Embed", "Adaptors\Embed\src\ArmoniK.Core.Adapters.Embed.csproj", "{F95A1BF7-B660-4789-9F2D-1749FC104EEE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArmoniK.Core.Adapters.Embed.Tests", "Adaptors\Embed\tests\ArmoniK.Core.Adapters.Embed.Tests.csproj", "{2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -413,6 +417,22 @@ Global {399C779C-CE8D-4757-8098-7F055467D96C}.Release|Any CPU.Build.0 = Release|Any CPU {399C779C-CE8D-4757-8098-7F055467D96C}.Release|x64.ActiveCfg = Release|Any CPU {399C779C-CE8D-4757-8098-7F055467D96C}.Release|x64.Build.0 = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|x64.ActiveCfg = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|x64.Build.0 = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|Any CPU.Build.0 = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|x64.ActiveCfg = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|x64.Build.0 = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|x64.ActiveCfg = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|x64.Build.0 = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|Any CPU.Build.0 = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|x64.ActiveCfg = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -465,6 +485,8 @@ Global {9F19E6DD-D9CD-43EC-B5CC-0081997ED8B8} = {6EE499A5-760F-4823-8AB6-DBDE8C5B5F45} {6ABF29BB-2F42-4088-AE93-E41416CB3271} = {CD412C3D-63D0-4726-B4C3-FEF701E4DCAF} {399C779C-CE8D-4757-8098-7F055467D96C} = {1A9BCE53-79D0-4761-B3A2-6967B610FA94} + {F95A1BF7-B660-4789-9F2D-1749FC104EEE} = {1A9BCE53-79D0-4761-B3A2-6967B610FA94} + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E} = {2D548C40-6260-4A4E-9C56-E8A80C639E13} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1285A466-2AF6-43E6-8DCC-2F93A5D5F02E} diff --git a/Base/src/ArmoniK.Core.Base.csproj b/Base/src/ArmoniK.Core.Base.csproj index e03c5c888..f5f3e59a7 100644 --- a/Base/src/ArmoniK.Core.Base.csproj +++ b/Base/src/ArmoniK.Core.Base.csproj @@ -30,8 +30,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + diff --git a/Adaptors/MongoDB/src/Options/ObjectStorage.cs b/Base/src/DataStructures/ObjectData.cs similarity index 65% rename from Adaptors/MongoDB/src/Options/ObjectStorage.cs rename to Base/src/DataStructures/ObjectData.cs index d50a98e53..0a6719d77 100644 --- a/Adaptors/MongoDB/src/Options/ObjectStorage.cs +++ b/Base/src/DataStructures/ObjectData.cs @@ -1,4 +1,4 @@ -// This file is part of the ArmoniK project +// This file is part of the ArmoniK project // // Copyright (C) ANEO, 2021-2024. All rights reserved. // @@ -15,14 +15,20 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using JetBrains.Annotations; +namespace ArmoniK.Core.Base.DataStructures; -namespace ArmoniK.Core.Adapters.MongoDB.Options; - -[PublicAPI] -public class ObjectStorage +/// +/// Metadata given to the +/// +public record ObjectData { - public const string SettingSection = nameof(MongoDB) + ":" + nameof(ObjectStorage); + /// + /// Id of the result + /// + public required string ResultId; - public int ChunkSize { get; set; } = 14500000; + /// + /// Id of the session + /// + public required string SessionId; } diff --git a/Common/src/Exceptions/ArmoniKException.cs b/Base/src/Exceptions/ArmoniKException.cs similarity index 97% rename from Common/src/Exceptions/ArmoniKException.cs rename to Base/src/Exceptions/ArmoniKException.cs index 1412f565b..339229974 100644 --- a/Common/src/Exceptions/ArmoniKException.cs +++ b/Base/src/Exceptions/ArmoniKException.cs @@ -17,7 +17,7 @@ using System; -namespace ArmoniK.Core.Common.Exceptions; +namespace ArmoniK.Core.Base.Exceptions; /// /// Base exception for exceptions in ArmoniK core diff --git a/Common/src/Exceptions/ObjectDataNotFoundException.cs b/Base/src/Exceptions/ObjectDataNotFoundException.cs similarity index 96% rename from Common/src/Exceptions/ObjectDataNotFoundException.cs rename to Base/src/Exceptions/ObjectDataNotFoundException.cs index bcf9ebe07..7a8c1ba4f 100644 --- a/Common/src/Exceptions/ObjectDataNotFoundException.cs +++ b/Base/src/Exceptions/ObjectDataNotFoundException.cs @@ -17,7 +17,7 @@ using System; -namespace ArmoniK.Core.Common.Exceptions; +namespace ArmoniK.Core.Base.Exceptions; [Serializable] public class ObjectDataNotFoundException : ArmoniKException diff --git a/Common/src/Storage/IObjectStorage.cs b/Base/src/IObjectStorage.cs similarity index 68% rename from Common/src/Storage/IObjectStorage.cs rename to Base/src/IObjectStorage.cs index 87315c2c4..13257773f 100644 --- a/Common/src/Storage/IObjectStorage.cs +++ b/Base/src/IObjectStorage.cs @@ -20,10 +20,10 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; -namespace ArmoniK.Core.Common.Storage; +namespace ArmoniK.Core.Base; /// /// Object Storage interface @@ -35,47 +35,40 @@ public interface IObjectStorage : IInitializable /// Add the given data in the storage at the given key /// Update data if it already exists /// - /// Key representing the object /// Chunks of data that will be stored in the Object Storage /// Token used to cancel the execution of the method /// - /// The size of the object that has been uploaded. + /// The opaque unique identifier representing the object and the size of the object that has been uploaded. /// + /// + /// Opaque ID will be used to refer to the object. The implementation of this interface should generate it. + /// /// the key is not found - Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default); + Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData data, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default); /// /// Get object in the Object Storage /// - /// Key representing the object to be retrieved + /// Opaque unique identifier representing the object /// Token used to cancel the execution of the method /// /// Byte arrays representing the object chunked /// /// the key is not found - IAsyncEnumerable GetValuesAsync(string key, + IAsyncEnumerable GetValuesAsync(byte[] id, CancellationToken cancellationToken = default); /// /// Delete data in the object storage /// - /// Keys representing the objects to delete + /// Opaque unique identifiers representing the objects to delete /// Token used to cancel the execution of the method /// /// Task representing the asynchronous execution of the method /// /// the key is not found - Task TryDeleteAsync(IEnumerable keys, + Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default); - - /// - /// List data in the Object Storage - /// - /// Token used to cancel the execution of the method - /// - /// The keys representing data found in the Object Storage - /// - IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default); } diff --git a/Common/src/ArmoniK.Core.Common.csproj b/Common/src/ArmoniK.Core.Common.csproj index 1b57bdf5e..3d77842c9 100644 --- a/Common/src/ArmoniK.Core.Common.csproj +++ b/Common/src/ArmoniK.Core.Common.csproj @@ -26,23 +26,23 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - - - + + + - - - - + + + + diff --git a/Common/src/Auth/Authentication/Authenticator.cs b/Common/src/Auth/Authentication/Authenticator.cs index 250176ee5..494ee2975 100644 --- a/Common/src/Auth/Authentication/Authenticator.cs +++ b/Common/src/Auth/Authentication/Authenticator.cs @@ -22,8 +22,8 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authorization; -using ArmoniK.Core.Common.Exceptions; using JetBrains.Annotations; diff --git a/Common/src/DynamicLoading/CollocatedAssemblyResolver.cs b/Common/src/DynamicLoading/CollocatedAssemblyResolver.cs new file mode 100644 index 000000000..8aa356679 --- /dev/null +++ b/Common/src/DynamicLoading/CollocatedAssemblyResolver.cs @@ -0,0 +1,106 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Reflection; + +using Microsoft.Extensions.Logging; + +namespace ArmoniK.Core.Common.DynamicLoading; + +public class CollocatedAssemblyResolver +{ + private static readonly ConcurrentDictionary PathDictionary = new(); + private readonly ILogger logger_; + + + public CollocatedAssemblyResolver(ILogger logger) + { + GetLoadDirectories(); + logger_ = logger; + } + + private static void GetLoadDirectories() + { + AppDomain.CurrentDomain.AssemblyLoad += (_, + eventArgs) => + { + var path = Path.GetDirectoryName(eventArgs.LoadedAssembly.Location); + if (path == null) + { + return; + } + + PathDictionary[path] = true; + }; + + PathDictionary[Path.GetDirectoryName(Assembly.GetCallingAssembly() + .Location)!] = true; + + PathDictionary[Path.GetDirectoryName(typeof(CollocatedAssemblyResolver).Assembly.Location)!] = true; + } + + public Assembly? AssemblyResolve(object? sender, + ResolveEventArgs args) + { + logger_.LogDebug("RequestingAssembly {RequestingAssembly}", + args.RequestingAssembly); + + if (args.RequestingAssembly?.Location is null) + { + return null; + } + + var assemblyName = new AssemblyName(args.Name).Name; + + if (string.IsNullOrEmpty(assemblyName)) + { + logger_.LogWarning("The assembly to resolve {AssemblyToResolve} does not have a name", + args.Name); + return null; + } + + var directoryName = Path.GetDirectoryName(args.RequestingAssembly.Location); + + if (!string.IsNullOrEmpty(directoryName)) + { + PathDictionary[directoryName] = true; + } + + foreach (var path in PathDictionary.Keys) + { + var assemblyPath = Path.Combine(path, + assemblyName + ".dll"); + + if (File.Exists(assemblyPath)) + { + logger_.LogDebug("Loading assembly {Assembly} from {Directory}", + assemblyName, + path); + return Assembly.LoadFile(assemblyPath); + } + + logger_.LogDebug("Assembly {Assembly} not found in {Directory}", + assemblyName, + path); + } + + return null; + } +} diff --git a/Common/src/Exceptions/InvalidSessionTransitionException.cs b/Common/src/Exceptions/InvalidSessionTransitionException.cs index 79abb5156..9d7a6c9a5 100644 --- a/Common/src/Exceptions/InvalidSessionTransitionException.cs +++ b/Common/src/Exceptions/InvalidSessionTransitionException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/PartitionNotFoundException.cs b/Common/src/Exceptions/PartitionNotFoundException.cs index 257c8d57a..bace4fc57 100644 --- a/Common/src/Exceptions/PartitionNotFoundException.cs +++ b/Common/src/Exceptions/PartitionNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/ResultNotFoundException.cs b/Common/src/Exceptions/ResultNotFoundException.cs index 74ce230f3..6d76cc1a3 100644 --- a/Common/src/Exceptions/ResultNotFoundException.cs +++ b/Common/src/Exceptions/ResultNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/SessionNotFoundException.cs b/Common/src/Exceptions/SessionNotFoundException.cs index c58e67f09..443f19675 100644 --- a/Common/src/Exceptions/SessionNotFoundException.cs +++ b/Common/src/Exceptions/SessionNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/SubmissionClosedException.cs b/Common/src/Exceptions/SubmissionClosedException.cs index 2894f5e9c..89b2db80a 100644 --- a/Common/src/Exceptions/SubmissionClosedException.cs +++ b/Common/src/Exceptions/SubmissionClosedException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskAlreadyExistsException.cs b/Common/src/Exceptions/TaskAlreadyExistsException.cs index 8dcd080c7..6cf3003a9 100644 --- a/Common/src/Exceptions/TaskAlreadyExistsException.cs +++ b/Common/src/Exceptions/TaskAlreadyExistsException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs b/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs index 0df0b44c6..5c6c98d3a 100644 --- a/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs +++ b/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskCanceledException.cs b/Common/src/Exceptions/TaskCanceledException.cs index eab3eb11b..ba7258efe 100644 --- a/Common/src/Exceptions/TaskCanceledException.cs +++ b/Common/src/Exceptions/TaskCanceledException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskNotFoundException.cs b/Common/src/Exceptions/TaskNotFoundException.cs index fa0851a8a..fe348ba75 100644 --- a/Common/src/Exceptions/TaskNotFoundException.cs +++ b/Common/src/Exceptions/TaskNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskPausedException.cs b/Common/src/Exceptions/TaskPausedException.cs index 474f33049..8e959912f 100644 --- a/Common/src/Exceptions/TaskPausedException.cs +++ b/Common/src/Exceptions/TaskPausedException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TimeoutException.cs b/Common/src/Exceptions/TimeoutException.cs index 06042151a..762f9f365 100644 --- a/Common/src/Exceptions/TimeoutException.cs +++ b/Common/src/Exceptions/TimeoutException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Injection/AdapterLoadContext.cs b/Common/src/Injection/AdapterLoadContext.cs deleted file mode 100644 index f3cfe3544..000000000 --- a/Common/src/Injection/AdapterLoadContext.cs +++ /dev/null @@ -1,54 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System.Reflection; -using System.Runtime.Loader; - -namespace ArmoniK.Core.Common.Injection; - -/// -/// Class holding load context for Adapters -/// -public class AdapterLoadContext : AssemblyLoadContext -{ - private readonly AssemblyDependencyResolver resolver_; - - /// - /// Instantiate a with the path to the assembly to be loaded. - /// - /// Path to the assembly - public AdapterLoadContext(string assemblyPath) - => resolver_ = new AssemblyDependencyResolver(assemblyPath); - - /// - protected override Assembly? Load(AssemblyName assemblyName) - { - var assemblyPath = resolver_.ResolveAssemblyToPath(assemblyName); - return assemblyPath is not null - ? LoadFromAssemblyPath(assemblyPath) - : null; - } - - /// - protected override nint LoadUnmanagedDll(string unmanagedDllName) - { - var libraryPath = resolver_.ResolveUnmanagedDllToPath(unmanagedDllName); - return libraryPath is not null - ? LoadUnmanagedDllFromPath(libraryPath) - : nint.Zero; - } -} diff --git a/Common/src/Injection/Options/Components.cs b/Common/src/Injection/Options/Components.cs index 905666df0..835134dc6 100644 --- a/Common/src/Injection/Options/Components.cs +++ b/Common/src/Injection/Options/Components.cs @@ -43,7 +43,7 @@ public class Components /// /// Represents which object storage is used to store data for tasks /// - public string? ObjectStorage { get; set; } + public AdapterSettings ObjectStorageAdaptorSettings { get; set; } = new(); /// /// Represents which database is used for authentication diff --git a/Common/src/Injection/ServiceCollectionExt.cs b/Common/src/Injection/ServiceCollectionExt.cs index ee56fc356..3799e182b 100644 --- a/Common/src/Injection/ServiceCollectionExt.cs +++ b/Common/src/Injection/ServiceCollectionExt.cs @@ -80,35 +80,37 @@ public static IServiceCollection AddArmoniKWorkerConnection(this IServiceCollect return services; } - public static IServiceCollection AddQueue(this IServiceCollection services, - ConfigurationManager configuration, - ILogger logger) + public static IServiceCollection AddAdapter(this IServiceCollection services, + ConfigurationManager configuration, + string storage, + ILogger logger) { - var queueSettings = configuration.GetRequiredValue($"{Components.SettingSection}:{nameof(Components.QueueAdaptorSettings)}"); + var settings = configuration.GetRequiredValue($"{Components.SettingSection}:{storage}"); - logger.LogInformation("Queue settings for loading adapter {@queueSettings}", - queueSettings); + logger.LogInformation("{storage} settings for loading adapter {@settings}", + storage, + settings); logger.LogDebug("{path}", - queueSettings.AdapterAbsolutePath); + settings.AdapterAbsolutePath); logger.LogDebug("{class}", - queueSettings.ClassName); + settings.ClassName); - if (string.IsNullOrEmpty(queueSettings.AdapterAbsolutePath)) + if (string.IsNullOrEmpty(settings.AdapterAbsolutePath)) { - throw new InvalidOperationException($"{nameof(queueSettings.AdapterAbsolutePath)} should not be null or empty."); + throw new InvalidOperationException($"{nameof(settings.AdapterAbsolutePath)} should not be null or empty."); } - if (string.IsNullOrEmpty(queueSettings.ClassName)) + if (string.IsNullOrEmpty(settings.ClassName)) { - throw new InvalidOperationException($"{nameof(queueSettings.ClassName)} should not be null or empty."); + throw new InvalidOperationException($"{nameof(settings.ClassName)} should not be null or empty."); } - var ctx = new AdapterLoadContext(queueSettings.AdapterAbsolutePath); - var assembly = ctx.LoadFromAssemblyName(AssemblyName.GetAssemblyName(queueSettings.AdapterAbsolutePath)); + var assembly = Assembly.LoadFrom(settings.AdapterAbsolutePath); + logger.LogInformation("Loaded assembly {assemblyName}", assembly.FullName); - var type = assembly.GetType(queueSettings.ClassName, + var type = assembly.GetType(settings.ClassName, true, true); diff --git a/Common/src/Pollster/AcquisitionStatus.cs b/Common/src/Pollster/AcquisitionStatus.cs index 3ba80e280..7e515edc4 100644 --- a/Common/src/Pollster/AcquisitionStatus.cs +++ b/Common/src/Pollster/AcquisitionStatus.cs @@ -98,6 +98,15 @@ public enum AcquisitionStatus /// TaskIsRetriedAndRetryIsSubmitted, + + /// + /// Task not acquired because its status is . Moreover, the retried task is + /// + /// Reinsertion in the queue may be required. + /// + TaskIsRetriedAndRetryIsPending, + + /// /// Task not acquired because its status is but the other pod does not seem to be /// processing it diff --git a/Common/src/Pollster/Agent.cs b/Common/src/Pollster/Agent.cs index c20973293..780b840c1 100644 --- a/Common/src/Pollster/Agent.cs +++ b/Common/src/Pollster/Agent.cs @@ -26,6 +26,8 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Api.gRPC.V1; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; @@ -45,16 +47,16 @@ namespace ArmoniK.Core.Common.Pollster; /// public sealed class Agent : IAgent { - private readonly List createdTasks_; - private readonly ILogger logger_; - private readonly IObjectStorage objectStorage_; - private readonly IPushQueueStorage pushQueueStorage_; - private readonly IResultTable resultTable_; - private readonly Dictionary sentResults_; - private readonly SessionData sessionData_; - private readonly ISubmitter submitter_; - private readonly TaskData taskData_; - private readonly ITaskTable taskTable_; + private readonly List createdTasks_; + private readonly ILogger logger_; + private readonly IObjectStorage objectStorage_; + private readonly IPushQueueStorage pushQueueStorage_; + private readonly IResultTable resultTable_; + private readonly Dictionary sentResults_; + private readonly SessionData sessionData_; + private readonly ISubmitter submitter_; + private readonly TaskData taskData_; + private readonly ITaskTable taskTable_; /// /// Initializes a new instance of the @@ -64,8 +66,8 @@ public sealed class Agent : IAgent /// Interface to put tasks in the queue /// Interface to manage result states /// Interface to manage task states - /// Data of the session - /// Data of the task + /// OpaqueId of the session + /// OpaqueId of the task /// Shared folder between Agent and Worker /// Token send to the worker to identify the running task /// Logger used to produce logs for this class @@ -87,7 +89,7 @@ public Agent(ISubmitter submitter, taskTable_ = taskTable; logger_ = logger; createdTasks_ = new List(); - sentResults_ = new Dictionary(); + sentResults_ = new Dictionary(); sessionData_ = sessionData; taskData_ = taskData; Folder = folder; @@ -120,11 +122,12 @@ await submitter_.FinalizeTaskCreation(createdTasks_, cancellationToken) .ConfigureAwait(false); - foreach (var (result, size) in sentResults_) + foreach (var (result, (id, size)) in sentResults_) { await resultTable_.CompleteResult(taskData_.SessionId, result, size, + id, cancellationToken) .ConfigureAwait(false); } @@ -155,29 +158,32 @@ public async Task GetResourceData(string token, ThrowIfInvalidToken(token); + try { - await using (var fs = new FileStream(Path.Combine(Folder, - resultId), - FileMode.OpenOrCreate)) + var opaqueId = (await resultTable_.GetResult(resultId, + cancellationToken) + .ConfigureAwait(false)).OpaqueId; + + await using var fs = new FileStream(Path.Combine(Folder, + resultId), + FileMode.OpenOrCreate); + await using var w = new BinaryWriter(fs); + await foreach (var chunk in objectStorage_.GetValuesAsync(opaqueId, + cancellationToken) + .ConfigureAwait(false)) { - await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(resultId, - cancellationToken) - .ConfigureAwait(false)) - { - w.Write(chunk); - } + w.Write(chunk); } return resultId; } - catch (ObjectDataNotFoundException) + catch (Exception ex) when (ex is ResultNotFoundException or ObjectDataNotFoundException) { throw new RpcException(new Status(StatusCode.NotFound, - "Data not found"), - "Data not found"); + "ResultId not found"), + "ResultId not found"); } } @@ -209,21 +215,6 @@ public Task GetDirectData(string token, throw new NotImplementedException("Direct data are not implemented yet"); } - /// - public async Task CancelChildTasks(CancellationToken cancellationToken) - { - if (createdTasks_.Any()) - { - await taskTable_.CancelTaskAsync(createdTasks_.Select(request => request.TaskId) - .AsICollection(), - cancellationToken) - .ConfigureAwait(false); - } - - logger_.LogDebug("Cancel {n} child tasks created by this task", - createdTasks_.Count); - } - /// public async Task> SubmitTasks(ICollection requests, TaskOptions? taskOptions, @@ -277,42 +268,48 @@ public async Task> CreateResults(string var results = await requests.Select(async rc => { - var resultId = Guid.NewGuid() - .ToString(); - - var size = await objectStorage_.AddOrUpdateAsync(resultId, - new List> - { - rc.data, - }.ToAsyncEnumerable(), - cancellationToken) - .ConfigureAwait(false); - - return new Result(rc.request.SessionId, - resultId, - rc.request.Name, - taskData_.TaskId, - "", - ResultStatus.Created, - new List(), - DateTime.UtcNow, - size, - Array.Empty()); + var result = new Result(rc.request.SessionId, + Guid.NewGuid() + .ToString(), + rc.request.Name, + taskData_.TaskId, + "", + ResultStatus.Created, + new List(), + DateTime.UtcNow, + 0, + Array.Empty()); + + var add = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = result.ResultId, + SessionId = rc.request.SessionId, + }, + new List> + { + rc.data, + }.ToAsyncEnumerable(), + cancellationToken) + .ConfigureAwait(false); + + return (result, add); }) .WhenAll() .ConfigureAwait(false); - await resultTable_.Create(results, + var ids = results.ViewSelect(tuple => tuple.result); + + await resultTable_.Create(ids, cancellationToken) .ConfigureAwait(false); foreach (var result in results) { - sentResults_.Add(result.ResultId, - result.Size); + sentResults_.Add(result.result.ResultId, + result.add); } - return results; + return ids; } /// @@ -330,19 +327,21 @@ public async Task> NotifyResultData(string toke using var r = new BinaryReader(fs); var channel = Channel.CreateUnbounded>(); - var add = objectStorage_.AddOrUpdateAsync(result, - channel.Reader.ReadAllAsync(cancellationToken), - cancellationToken); + var addTask = objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = result, + SessionId = sessionData_.SessionId, + }, + channel.Reader.ReadAllAsync(cancellationToken), + cancellationToken); - long size = 0; - int read; + int read; do { var buffer = new byte[PayloadConfiguration.MaxChunkSize]; read = r.Read(buffer, 0, PayloadConfiguration.MaxChunkSize); - size += read; if (read > 0) { await channel.Writer.WriteAsync(buffer.AsMemory(0, @@ -354,9 +353,9 @@ await channel.Writer.WriteAsync(buffer.AsMemory(0, channel.Writer.Complete(); - await add.ConfigureAwait(false); + var add = await addTask.ConfigureAwait(false); sentResults_.Add(result, - size); + add); } return resultIds; diff --git a/Common/src/Pollster/DataPrefetcher.cs b/Common/src/Pollster/DataPrefetcher.cs index cb6ca1f5a..ae566f9f6 100644 --- a/Common/src/Pollster/DataPrefetcher.cs +++ b/Common/src/Pollster/DataPrefetcher.cs @@ -15,6 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Threading; @@ -23,7 +24,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -39,6 +40,7 @@ public class DataPrefetcher : IInitializable private readonly ActivitySource? activitySource_; private readonly ILogger logger_; private readonly IObjectStorage objectStorage_; + private readonly IResultTable resultTable_; private bool isInitialized_; @@ -46,13 +48,16 @@ public class DataPrefetcher : IInitializable /// Create data prefetcher for tasks /// /// Interface to manage data + /// Interface to manage results metadata /// Activity source for tracing /// Logger used to print logs public DataPrefetcher(IObjectStorage objectStorage, + IResultTable resultTable, ActivitySource? activitySource, ILogger logger) { objectStorage_ = objectStorage; + resultTable_ = resultTable; logger_ = logger; activitySource_ = activitySource; } @@ -93,28 +98,27 @@ public async Task PrefetchDataAsync(TaskData taskData, activity?.AddEvent(new ActivityEvent("Load payload")); - - await using (var fs = new FileStream(Path.Combine(folder, - taskData.PayloadId), - FileMode.OpenOrCreate)) - { - await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(taskData.PayloadId, - cancellationToken) - .ConfigureAwait(false)) - { - w.Write(chunk); - } - } + var dependencies = new List + { + taskData.PayloadId, + }; + dependencies.AddRange(taskData.DataDependencies); - foreach (var dataDependency in taskData.DataDependencies) + await foreach (var id in resultTable_.GetResults(data => dependencies.Contains(data.ResultId), + r => new + { + r.ResultId, + r.OpaqueId, + }, + cancellationToken) + .ConfigureAwait(false)) { await using var fs = new FileStream(Path.Combine(folder, - dataDependency), + id.ResultId), FileMode.OpenOrCreate); await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(dataDependency, + await foreach (var chunk in objectStorage_.GetValuesAsync(id.OpaqueId, cancellationToken) .ConfigureAwait(false)) { diff --git a/Common/src/Pollster/IAgent.cs b/Common/src/Pollster/IAgent.cs index bff4a4806..82e033bcb 100644 --- a/Common/src/Pollster/IAgent.cs +++ b/Common/src/Pollster/IAgent.cs @@ -145,13 +145,4 @@ Task> CreateResults(string Task> NotifyResultData(string token, ICollection resultIds, CancellationToken cancellationToken); - - /// - /// Cancel child tasks created by the current task in processing - /// - /// Token used to cancel the execution of the method - /// - /// Task representing the asynchronous execution of the method - /// - Task CancelChildTasks(CancellationToken cancellationToken); } diff --git a/Common/src/Pollster/Pollster.cs b/Common/src/Pollster/Pollster.cs index 4f869cc49..2ae52e068 100644 --- a/Common/src/Pollster/Pollster.cs +++ b/Common/src/Pollster/Pollster.cs @@ -372,14 +372,27 @@ await messages.Current.DisposeIgnoreErrorAsync(logger_) await taskHandler.PreProcessing() .ConfigureAwait(false); - await runningTaskQueue_.WriteAsync(taskHandler, - pollsterOptions_.TimeoutBeforeNextAcquisition, - exceptionManager_.EarlyCancellationToken) - .ConfigureAwait(false); - - // TaskHandler has been successfully sent to the next stage of the pipeline - // So remove the automatic dispose of the TaskHandler - taskHandlerDispose.Reset(); + try + { + await runningTaskQueue_.WriteAsync(taskHandler, + pollsterOptions_.TimeoutBeforeNextAcquisition, + exceptionManager_.EarlyCancellationToken) + .ConfigureAwait(false); + + // TaskHandler has been successfully sent to the next stage of the pipeline + // So remove the automatic dispose of the TaskHandler + taskHandlerDispose.Reset(); + } + catch (Exception e) + { + await taskHandler.ReleaseAndPostponeTask() + .ConfigureAwait(false); + + if (e is not (TimeoutException or OperationCanceledException)) + { + throw; + } + } } catch (Exception e) { diff --git a/Common/src/Pollster/TaskHandler.cs b/Common/src/Pollster/TaskHandler.cs index cb6ae1329..73daabfc8 100644 --- a/Common/src/Pollster/TaskHandler.cs +++ b/Common/src/Pollster/TaskHandler.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Meter; @@ -480,7 +481,7 @@ await taskTable_.RetryTask(taskData_, .ConfigureAwait(false); } - if (retryData.Status is TaskStatus.Creating or TaskStatus.Submitted) + if (retryData.Status is TaskStatus.Creating or TaskStatus.Pending or TaskStatus.Submitted) { logger_.LogWarning("Retried task {task} is in {status}; trying to finalize task creation", retryId, @@ -498,11 +499,18 @@ await submitter_.FinalizeTaskCreation(new List CancellationToken.None) .ConfigureAwait(false); } + else + { + logger_.LogInformation("Retried task {task} is in {status}; nothing done", + retryId, + retryData.Status); + } return (taskNotFound, taskExists, retryData.Status) switch { (false, false, TaskStatus.Submitted) => AcquisitionStatus.TaskIsRetriedAndRetryIsSubmitted, (false, false, TaskStatus.Creating) => AcquisitionStatus.TaskIsRetriedAndRetryIsCreating, + (false, false, TaskStatus.Pending) => AcquisitionStatus.TaskIsRetriedAndRetryIsPending, (true, false, TaskStatus.Submitted or TaskStatus.Creating) => AcquisitionStatus.TaskIsRetriedAndRetryIsNotFound, _ => AcquisitionStatus.TaskIsRetried, }; @@ -960,11 +968,6 @@ public async Task PostProcessing() await agent_.FinalizeTaskCreation(CancellationToken.None) .ConfigureAwait(false); } - else - { - await agent_.CancelChildTasks(CancellationToken.None) - .ConfigureAwait(false); - } await submitter_.CompleteTaskAsync(taskData_, sessionData_, @@ -1089,12 +1092,6 @@ await submitter_.CompleteTaskAsync(taskData, ? QueueMessageStatus.Cancelled : QueueMessageStatus.Processed; } - - if (agent_ is not null) - { - await agent_.CancelChildTasks(CancellationToken.None) - .ConfigureAwait(false); - } } // Rethrow enable the recording of the error by the Pollster Main loop diff --git a/Common/src/Storage/Result.cs b/Common/src/Storage/Result.cs index f0975b352..bb62abd48 100644 --- a/Common/src/Storage/Result.cs +++ b/Common/src/Storage/Result.cs @@ -31,7 +31,7 @@ namespace ArmoniK.Core.Common.Storage; /// List of tasks that depend on this result. /// Date of creation of the current object. /// Size of the result. -/// Data for the current +/// Opaque Identifier used by the object storage to refer to this result's data public record Result(string SessionId, string ResultId, string Name, @@ -41,7 +41,7 @@ public record Result(string SessionId, List DependentTasks, DateTime CreationDate, long Size, - byte[] Data) + byte[] OpaqueId) { /// /// Creates a copy of a and modify it according to given updates diff --git a/Common/src/Storage/ResultLifeCycleHelper.cs b/Common/src/Storage/ResultLifeCycleHelper.cs index 81e630e40..36c8d22d9 100644 --- a/Common/src/Storage/ResultLifeCycleHelper.cs +++ b/Common/src/Storage/ResultLifeCycleHelper.cs @@ -22,6 +22,7 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Utils; using Microsoft.Extensions.Logging; @@ -156,8 +157,8 @@ public static async Task PurgeResultsAsync(IResultTable resultTable, string sessionId, CancellationToken cancellationToken) { - await foreach (var ids in resultTable.GetResults(result => result.SessionId == sessionId, - result => result.ResultId, + await foreach (var ids in resultTable.GetResults(result => result.SessionId == sessionId && result.OpaqueId.Length > 0, + result => result.OpaqueId, cancellationToken) .ToChunksAsync(500, Timeout.InfiniteTimeSpan, @@ -171,7 +172,9 @@ await objectStorage.TryDeleteAsync(ids, await resultTable.UpdateManyResults(result => result.SessionId == sessionId, new UpdateDefinition().Set(result => result.Status, - ResultStatus.DeletedData), + ResultStatus.DeletedData) + .Set(result => result.OpaqueId, + Array.Empty()), cancellationToken) .ConfigureAwait(false); } diff --git a/Common/src/Storage/ResultTableExtensions.cs b/Common/src/Storage/ResultTableExtensions.cs index 08df40cd7..cbb0506c8 100644 --- a/Common/src/Storage/ResultTableExtensions.cs +++ b/Common/src/Storage/ResultTableExtensions.cs @@ -55,6 +55,29 @@ await resultTable.UpdateManyResults(result => result.OwnerTaskId == ownerTaskId, ownerTaskId); } + /// + /// Abort the results of the given session + /// + /// Interface to manage results + /// id of the session containing the results + /// Token used to cancel the execution of the method + /// + /// Task representing the asynchronous execution of the method + /// + public static async Task AbortSessionResults(this IResultTable resultTable, + string sessionId, + CancellationToken cancellationToken = default) + { + await resultTable.UpdateManyResults(result => result.SessionId == sessionId && result.Status == ResultStatus.Created, + new UpdateDefinition().Set(data => data.Status, + ResultStatus.Aborted), + cancellationToken) + .ConfigureAwait(false); + + resultTable.Logger.LogDebug("Abort results from {session}", + sessionId); + } + /// /// Updates in bulk results @@ -78,6 +101,7 @@ public static Task BulkUpdateResults(this IResultTable /// id of the session containing the results /// Id of the result to complete /// Size of the result to complete + /// Opaque unique identifier representing the object /// Token used to cancel the execution of the method /// /// The new version of the result metadata @@ -87,13 +111,16 @@ public static async Task CompleteResult(this IResultTable resultTable, string sessionId, string resultId, long size, + byte[] opaqueId, CancellationToken cancellationToken = default) { var result = await resultTable.UpdateOneResult(resultId, new UpdateDefinition().Set(data => data.Status, ResultStatus.Completed) .Set(data => data.Size, - size), + size) + .Set(data => data.OpaqueId, + opaqueId), cancellationToken) .ConfigureAwait(false); @@ -105,49 +132,10 @@ public static async Task CompleteResult(this IResultTable resultTable, { Status = ResultStatus.Completed, Size = size, + OpaqueId = opaqueId, }; } - /// - /// Update result with small payload - /// - /// Interface to manage results - /// id of the session containing the results - /// id of the task owning the result - /// id of the result to be modified - /// payload data - /// Token used to cancel the execution of the method - /// - /// Task representing the asynchronous execution of the method - /// - public static async Task SetResult(this IResultTable resultTable, - string sessionId, - string ownerTaskId, - string resultId, - byte[] smallPayload, - CancellationToken cancellationToken = default) - { - var count = await resultTable.UpdateManyResults(result => result.ResultId == resultId && result.OwnerTaskId == ownerTaskId, - new UpdateDefinition().Set(result => result.Status, - ResultStatus.Completed) - .Set(data => data.Size, - smallPayload.Length) - .Set(result => result.Data, - smallPayload), - cancellationToken) - .ConfigureAwait(false); - - resultTable.Logger.LogDebug("Update {result} from {owner} to {status}", - resultId, - ownerTaskId, - ResultStatus.Completed); - - if (count == 0) - { - throw new ResultNotFoundException($"Result '{resultId}' was not found for '{ownerTaskId}'"); - } - } - /// /// Update result /// @@ -165,13 +153,16 @@ public static async Task SetResult(this IResultTable resultTable, string ownerTaskId, string resultId, long size, + byte[] opaqueId, CancellationToken cancellationToken = default) { var count = await resultTable.UpdateManyResults(result => result.ResultId == resultId && result.OwnerTaskId == ownerTaskId, new UpdateDefinition().Set(result => result.Status, ResultStatus.Completed) .Set(result => result.Size, - size), + size) + .Set(result => result.OpaqueId, + opaqueId), cancellationToken) .ConfigureAwait(false); diff --git a/Common/src/Storage/SessionTableExtensions.cs b/Common/src/Storage/SessionTableExtensions.cs index 5996d958c..83582f4c5 100644 --- a/Common/src/Storage/SessionTableExtensions.cs +++ b/Common/src/Storage/SessionTableExtensions.cs @@ -233,7 +233,7 @@ public static async Task ResumeSessionAsync(this ISessionTable sess CancellationToken cancellationToken = default) { var session = await sessionTable.UpdateOneSessionAsync(sessionId, - data => data.Status == SessionStatus.Paused, + data => data.Status == SessionStatus.Paused || data.Status == SessionStatus.Running, new UpdateDefinition().Set(model => model.Status, SessionStatus.Running), false, @@ -251,8 +251,8 @@ public static async Task ResumeSessionAsync(this ISessionTable sess switch (session.Status) { case SessionStatus.Paused: - throw new UnreachableException($"Session status should be {SessionStatus.Running} but is {session.Status}"); case SessionStatus.Running: + throw new UnreachableException($"Session status should be {SessionStatus.Running} but is {session.Status}"); case SessionStatus.Purged: case SessionStatus.Cancelled: throw new InvalidSessionTransitionException($"Cannot resume a session with status {session.Status}"); diff --git a/Common/src/Storage/TaskData.cs b/Common/src/Storage/TaskData.cs index d80429b37..03fd9537f 100644 --- a/Common/src/Storage/TaskData.cs +++ b/Common/src/Storage/TaskData.cs @@ -18,7 +18,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; using ArmoniK.Core.Base.DataStructures; @@ -140,7 +139,7 @@ public TaskData(string sessionId, { payloadId, }) - .ToDictionary(EscapeKey, + .ToDictionary(s => s, _ => true), expectedOutputIds, taskId, @@ -175,23 +174,6 @@ public TaskData(TaskData original, : this(original) => updates.ApplyTo(this); - /// - /// ResultIds could contain dots (eg: it is the case in htcmock), - /// but MongoDB does not support well dots in keys. - /// This escapes the key to replace dots with something else. - /// Escaped keys are guaranteed to have neither dots nor dollars - /// - /// Key string - /// Escaped key - public static string EscapeKey(string key) - => new StringBuilder(key).Replace("@", - "@at@") - .Replace(".", - "@dot@") - .Replace("$", - "@dollar@") - .ToString(); - /// /// Conversion operator from to /// diff --git a/Common/src/Storage/TaskLifeCycleHelper.cs b/Common/src/Storage/TaskLifeCycleHelper.cs index 77e9a2697..99c337cb0 100644 --- a/Common/src/Storage/TaskLifeCycleHelper.cs +++ b/Common/src/Storage/TaskLifeCycleHelper.cs @@ -24,6 +24,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Utils; @@ -223,6 +224,9 @@ public static async Task FinalizeTaskCreation(ITaskTable t return; } + logger.LogDebug("Tasks to finalize : {@TaskRequests}", + taskRequests); + var prepareTaskDependencies = PrepareTaskDependencies(taskTable, resultTable, taskRequests, @@ -260,6 +264,7 @@ await EnqueueReadyTasks(taskTable, pushQueueStorage, sessionData, readyTask, + logger, cancellationToken) .ConfigureAwait(false); } @@ -447,6 +452,7 @@ await EnqueueReadyTasks(taskTable, pushQueueStorage, sessionData, readyTasks, + logger, cancellationToken) .ConfigureAwait(false); } @@ -466,6 +472,7 @@ private static async Task EnqueueReadyTasks(ITaskTable taskTable, IPushQueueStorage pushQueueStorage, SessionData sessionData, ICollection messages, + ILogger logger, CancellationToken cancellationToken) { if (!messages.Any()) @@ -482,6 +489,9 @@ await pushQueueStorage.PushMessagesAsync(group, cancellationToken) .ConfigureAwait(false); } + + logger.LogDebug("Pushed messages : {@Messages}", + messages); } await taskTable.FinalizeTaskCreation(messages.Select(task => task.TaskId) @@ -521,20 +531,27 @@ public static async Task ResumeAsync(ITaskTable taskTable, .WithCancellation(cancellationToken) .ConfigureAwait(false)) { - var tasks = await grouping.ToListAsync(cancellationToken) - .ConfigureAwait(false); + await foreach (var tasks in grouping.ToChunksAsync(100000, + TimeSpan.FromSeconds(1), + cancellationToken) + .ConfigureAwait(false)) + { + cancellationToken.ThrowIfCancellationRequested(); - var taskIds = tasks.Select(task => task.TaskId) - .AsICollection(); - await taskTable.UpdateManyTasks(data => data.SessionId == sessionId && data.Status == TaskStatus.Paused && taskIds.Contains(data.TaskId), - new UpdateDefinition().Set(data => data.Status, - TaskStatus.Submitted), - cancellationToken) - .ConfigureAwait(false); - await pushQueueStorage.PushMessagesAsync(tasks, - grouping.Key.PartitionId, - cancellationToken) - .ConfigureAwait(false); + var taskIds = tasks.Select(task => task.TaskId) + .AsICollection(); + + await taskTable.UpdateManyTasks(data => data.SessionId == sessionId && data.Status == TaskStatus.Paused && taskIds.Contains(data.TaskId), + new UpdateDefinition().Set(data => data.Status, + TaskStatus.Submitted), + CancellationToken.None) + .ConfigureAwait(false); + + await pushQueueStorage.PushMessagesAsync(tasks, + grouping.Key.PartitionId, + CancellationToken.None) + .ConfigureAwait(false); + } } return session; diff --git a/Common/src/Storage/TaskTableExtensions.cs b/Common/src/Storage/TaskTableExtensions.cs index 7a14a88be..e299bb68b 100644 --- a/Common/src/Storage/TaskTableExtensions.cs +++ b/Common/src/Storage/TaskTableExtensions.cs @@ -22,6 +22,7 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using Microsoft.Extensions.Logging; diff --git a/Common/src/Stream/Worker/WorkerStreamHandler.cs b/Common/src/Stream/Worker/WorkerStreamHandler.cs index 115de8541..0a5a209c8 100644 --- a/Common/src/Stream/Worker/WorkerStreamHandler.cs +++ b/Common/src/Stream/Worker/WorkerStreamHandler.cs @@ -23,7 +23,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Worker; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Convertors; using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Storage; diff --git a/Common/src/Utils/LoggerInit.cs b/Common/src/Utils/LoggerInit.cs index f05d16150..f354cad28 100644 --- a/Common/src/Utils/LoggerInit.cs +++ b/Common/src/Utils/LoggerInit.cs @@ -16,6 +16,7 @@ // along with this program. If not, see . using System; +using System.Diagnostics; using Destructurama; @@ -43,8 +44,8 @@ public LoggerInit(IConfiguration configuration) .WriteTo.Console(new CompactJsonFormatter()) .Enrich.FromLogContext() .Enrich.WithProperty("CoreVersion", - typeof(LoggerInit).Assembly.GetName() - .Version?.ToString() ?? "Unknown") + FileVersionInfo.GetVersionInfo(typeof(LoggerInit).Assembly.Location) + .ProductVersion ?? "Unknown") .Enrich.WithProperty("InstanceId", instanceId) .Destructure.UsingAttributes() diff --git a/Common/src/gRPC/Convertors/TaskDataHolderExt.cs b/Common/src/gRPC/Convertors/TaskDataHolderExt.cs index 5e968b0c5..f0af6f73f 100644 --- a/Common/src/gRPC/Convertors/TaskDataHolderExt.cs +++ b/Common/src/gRPC/Convertors/TaskDataHolderExt.cs @@ -136,9 +136,7 @@ public static TaskSummary ToTaskSummary(this TaskDataHolder taskDataSummary) StartedAt = taskDataSummary.StartDate is not null ? FromDateTime(taskDataSummary.StartDate.Value) : null, - Error = taskDataSummary.Status == TaskStatus.Error - ? taskDataSummary.Output?.Error - : "", + Error = taskDataSummary.Output?.Error ?? string.Empty, StatusMessage = taskDataSummary.StatusMessage, SubmittedAt = taskDataSummary.SubmittedDate is not null ? FromDateTime(taskDataSummary.SubmittedDate.Value) diff --git a/Common/src/gRPC/Convertors/TaskTableExt.cs b/Common/src/gRPC/Convertors/TaskTableExt.cs index 2f566c744..72bf7c68a 100644 --- a/Common/src/gRPC/Convertors/TaskTableExt.cs +++ b/Common/src/gRPC/Convertors/TaskTableExt.cs @@ -20,7 +20,7 @@ using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1.Submitter; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using TaskStatus = ArmoniK.Core.Common.Storage.TaskStatus; diff --git a/Common/src/gRPC/RpcExt.cs b/Common/src/gRPC/RpcExt.cs index 9885482f8..386be962e 100644 --- a/Common/src/gRPC/RpcExt.cs +++ b/Common/src/gRPC/RpcExt.cs @@ -1,4 +1,4 @@ -// This file is part of the ArmoniK project +// This file is part of the ArmoniK project // // Copyright (C) ANEO, 2021-2024. All rights reserved. // @@ -22,7 +22,7 @@ using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using Grpc.Core; diff --git a/Common/src/gRPC/Services/GrpcAuthService.cs b/Common/src/gRPC/Services/GrpcAuthService.cs index 78ee74944..9d1f0cb8e 100644 --- a/Common/src/gRPC/Services/GrpcAuthService.cs +++ b/Common/src/gRPC/Services/GrpcAuthService.cs @@ -20,10 +20,10 @@ using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1.Auth; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Auth.Authorization.Permissions; -using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Meter; using Grpc.Core; diff --git a/Common/src/gRPC/Services/GrpcResultsService.cs b/Common/src/gRPC/Services/GrpcResultsService.cs index e3cf424fc..8d99b6ccc 100644 --- a/Common/src/gRPC/Services/GrpcResultsService.cs +++ b/Common/src/gRPC/Services/GrpcResultsService.cs @@ -25,6 +25,8 @@ using ArmoniK.Api.gRPC.V1.Results; using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; @@ -171,35 +173,40 @@ public override async Task CreateResults(CreateResultsReq var resultId = Guid.NewGuid() .ToString(); - var size = await objectStorage_.AddOrUpdateAsync(resultId, - new List> - { - rc.Data.Memory, - }.ToAsyncEnumerable(), - context.CancellationToken) - .ConfigureAwait(false); - - return new Result(request.SessionId, - resultId, - rc.Name, - "", - request.SessionId, - ResultStatus.Created, - new List(), - DateTime.UtcNow, - size, - Array.Empty()); + var (id, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = resultId, + SessionId = request.SessionId, + }, + new List> + { + rc.Data.Memory, + }.ToAsyncEnumerable(), + context.CancellationToken) + .ConfigureAwait(false); + + return (new Result(request.SessionId, + resultId, + rc.Name, + "", + request.SessionId, + ResultStatus.Created, + new List(), + DateTime.UtcNow, + size, + Array.Empty()), id); }) .WhenAll() .ConfigureAwait(false); - await resultTable_.Create(results, + await resultTable_.Create(results.ViewSelect(tuple => tuple.Item1), context.CancellationToken) .ConfigureAwait(false); var resultList = await results.Select(async r => await resultTable_.CompleteResult(request.SessionId, - r.ResultId, - r.Size) + r.Item1.ResultId, + r.Item1.Size, + r.id) .ConfigureAwait(false)) .WhenAll() .ConfigureAwait(false); @@ -221,7 +228,13 @@ public override async Task DeleteResultsData(DeleteRe using var measure = meter_.CountAndTime(); try { - await objectStorage_.TryDeleteAsync(request.ResultId, + var opaqueIds = await resultTable_.GetResults(result => request.ResultId.Contains(result.ResultId), + result => result.OpaqueId, + context.CancellationToken) + .ToListAsync(context.CancellationToken) + .ConfigureAwait(false); + + await objectStorage_.TryDeleteAsync(opaqueIds, context.CancellationToken) .ConfigureAwait(false); @@ -236,7 +249,14 @@ await objectStorage_.TryDeleteAsync(request.ResultId, catch (ObjectDataNotFoundException e) { logger_.LogWarning(e, - "Error while downloading results"); + "Error while deleting results"); + throw new RpcException(new Status(StatusCode.NotFound, + "Result data not found")); + } + catch (ResultNotFoundException e) + { + logger_.LogWarning(e, + "Error while deleting results"); throw new RpcException(new Status(StatusCode.NotFound, "Result data not found")); } @@ -251,7 +271,10 @@ public override async Task DownloadResultData(DownloadResultDataRequest using var measure = meter_.CountAndTime(); try { - await foreach (var chunk in objectStorage_.GetValuesAsync(request.ResultId, + var id = (await resultTable_.GetResult(request.ResultId) + .ConfigureAwait(false)).OpaqueId; + + await foreach (var chunk in objectStorage_.GetValuesAsync(id, context.CancellationToken) .ConfigureAwait(false)) { @@ -269,6 +292,13 @@ await responseStream.WriteAsync(new DownloadResultDataResponse throw new RpcException(new Status(StatusCode.NotFound, "Result data not found")); } + catch (ResultNotFoundException e) + { + logger_.LogWarning(e, + "Error while downloading results"); + throw new RpcException(new Status(StatusCode.NotFound, + "Result data not found")); + } } [RequiresPermission(typeof(GrpcResultsService), @@ -311,17 +341,16 @@ public override async Task UploadResultData(IAsyncStre var sessionTask = sessionTable_.GetSessionAsync(id.SessionId, context.CancellationToken); - long size = 0; - await objectStorage_.AddOrUpdateAsync(id.ResultId, - requestStream.ReadAllAsync(context.CancellationToken) - .Select(r => - { - size += r.DataChunk.Length; - return r.DataChunk.Memory; - }), - context.CancellationToken) - .ConfigureAwait(false); + var (opaqueId, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = id.ResultId, + SessionId = id.SessionId, + }, + requestStream.ReadAllAsync(context.CancellationToken) + .Select(r => r.DataChunk.Memory), + context.CancellationToken) + .ConfigureAwait(false); try { @@ -330,6 +359,7 @@ await objectStorage_.AddOrUpdateAsync(id.ResultId, var resultData = await resultTable_.CompleteResult(id.SessionId, id.ResultId, size, + opaqueId, context.CancellationToken) .ConfigureAwait(false); diff --git a/Common/src/gRPC/Services/GrpcSessionsService.cs b/Common/src/gRPC/Services/GrpcSessionsService.cs index c65cf27fe..04fd4e829 100644 --- a/Common/src/gRPC/Services/GrpcSessionsService.cs +++ b/Common/src/gRPC/Services/GrpcSessionsService.cs @@ -23,6 +23,7 @@ using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; @@ -35,8 +36,6 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.Extensions.Logging; -using TaskStatus = ArmoniK.Core.Common.Storage.TaskStatus; - namespace ArmoniK.Core.Common.gRPC.Services; [Authorize(AuthenticationSchemes = Authenticator.SchemeName)] @@ -81,11 +80,18 @@ public override async Task CancelSession(CancelSessionReq using var measure = meter_.CountAndTime(); try { + var tasks = taskTable_.CancelSessionAsync(request.SessionId, + context.CancellationToken); + var results = resultTable_.AbortSessionResults(request.SessionId, + context.CancellationToken); + var sessions = sessionTable_.CancelSessionAsync(request.SessionId, + context.CancellationToken); + + await tasks.ConfigureAwait(false); + await results.ConfigureAwait(false); return new CancelSessionResponse { - Session = (await sessionTable_.CancelSessionAsync(request.SessionId, - context.CancellationToken) - .ConfigureAwait(false)).ToGrpcSessionRaw(), + Session = (await sessions.ConfigureAwait(false)).ToGrpcSessionRaw(), }; } catch (SessionNotFoundException e) @@ -509,36 +515,28 @@ public override async Task ResumeSession(ResumeSessionReq catch (SessionNotFoundException e) { logger_.LogWarning(e, - "Error while getting session"); + "Error while resuming session"); throw new RpcException(new Status(StatusCode.NotFound, "Session not found")); } catch (InvalidSessionTransitionException e) { logger_.LogWarning(e, - "Error while cancelling session"); + "Error while resuming session"); throw new RpcException(new Status(StatusCode.FailedPrecondition, "Session is in a state that cannot be cancelled")); } catch (ArmoniKException e) { logger_.LogWarning(e, - "Error while getting session"); - await taskTable_.UpdateManyTasks(data => data.SessionId == request.SessionId && data.Status == TaskStatus.Submitted, - new UpdateDefinition().Set(data => data.Status, - TaskStatus.Paused)) - .ConfigureAwait(false); + "Error while resuming session"); throw new RpcException(new Status(StatusCode.Internal, "Internal Armonik Exception, see application logs")); } catch (Exception e) { logger_.LogWarning(e, - "Error while getting session"); - await taskTable_.UpdateManyTasks(data => data.SessionId == request.SessionId && data.Status == TaskStatus.Submitted, - new UpdateDefinition().Set(data => data.Status, - TaskStatus.Paused)) - .ConfigureAwait(false); + "Error while resuming session"); throw new RpcException(new Status(StatusCode.Unknown, "Unknown Exception, see application logs")); } diff --git a/Common/src/gRPC/Services/GrpcSubmitterService.cs b/Common/src/gRPC/Services/GrpcSubmitterService.cs index 7c6c5f0d0..0ea46f946 100644 --- a/Common/src/gRPC/Services/GrpcSubmitterService.cs +++ b/Common/src/gRPC/Services/GrpcSubmitterService.cs @@ -22,6 +22,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; diff --git a/Common/src/gRPC/Services/GrpcTasksService.cs b/Common/src/gRPC/Services/GrpcTasksService.cs index 0118b5979..7f5527edb 100644 --- a/Common/src/gRPC/Services/GrpcTasksService.cs +++ b/Common/src/gRPC/Services/GrpcTasksService.cs @@ -25,6 +25,7 @@ using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Api.gRPC.V1.Tasks; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; diff --git a/Common/src/gRPC/Services/GrpcVersionsService.cs b/Common/src/gRPC/Services/GrpcVersionsService.cs index 4d39d368e..86852b51b 100644 --- a/Common/src/gRPC/Services/GrpcVersionsService.cs +++ b/Common/src/gRPC/Services/GrpcVersionsService.cs @@ -15,6 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using System.Diagnostics; using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1.Versions; @@ -30,11 +31,11 @@ namespace ArmoniK.Core.Common.gRPC.Services; [Authorize(AuthenticationSchemes = Authenticator.SchemeName)] public class GrpcVersionsService : Versions.VersionsBase { - public static readonly string CoreVersion = typeof(GrpcVersionsService).Assembly.GetName() - .Version!.ToString(); + public static readonly string CoreVersion = FileVersionInfo.GetVersionInfo(typeof(GrpcVersionsService).Assembly.Location) + .ProductVersion ?? "Unknown"; - public static readonly string ApiVersion = typeof(Versions.VersionsBase).Assembly.GetName() - .Version!.ToString(); + public static readonly string ApiVersion = FileVersionInfo.GetVersionInfo(typeof(Versions.VersionsBase).Assembly.Location) + .ProductVersion ?? "Unknown"; [RequiresPermission(typeof(GrpcVersionsService), nameof(ListVersions))] diff --git a/Common/src/gRPC/Services/Submitter.cs b/Common/src/gRPC/Services/Submitter.cs index 35e8522e5..fab7bfb2c 100644 --- a/Common/src/gRPC/Services/Submitter.cs +++ b/Common/src/gRPC/Services/Submitter.cs @@ -27,7 +27,8 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Convertors; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; @@ -196,7 +197,7 @@ await responseStream.WriteAsync(new ResultReply } } - await foreach (var chunk in objectStorage_.GetValuesAsync(request.ResultId, + await foreach (var chunk in objectStorage_.GetValuesAsync(result.OpaqueId, cancellationToken) .ConfigureAwait(false)) { @@ -394,15 +395,20 @@ public async Task SetResult(string sessionId, { using var activity = activitySource_.StartActivity($"{nameof(SetResult)}"); - var size = await objectStorage_.AddOrUpdateAsync(key, - chunks, - cancellationToken) - .ConfigureAwait(false); + var (id, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = key, + SessionId = sessionId, + }, + chunks, + cancellationToken) + .ConfigureAwait(false); await resultTable_.SetResult(sessionId, ownerTaskId, key, size, + id, cancellationToken) .ConfigureAwait(false); } @@ -432,7 +438,7 @@ public async Task> CreateTasks(string ("PartitionId", options.PartitionId)); var requests = new List(); - var payloadUploadTasks = new List>(); + var payloadUploadTasks = new List>(); await foreach (var taskRequest in taskRequests.WithCancellation(cancellationToken) .ConfigureAwait(false)) @@ -447,7 +453,11 @@ public async Task> CreateTasks(string options, taskRequest.ExpectedOutputKeys.ToList(), taskRequest.DataDependencies.ToList())); - payloadUploadTasks.Add(objectStorage_.AddOrUpdateAsync(payloadId, + payloadUploadTasks.Add(objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = payloadId, + SessionId = sessionId, + }, taskRequest.PayloadChunks, cancellationToken)); } @@ -457,18 +467,18 @@ public async Task> CreateTasks(string await resultTable_.Create(requests.Zip(payloadSizes, (request, - size) => new Result(sessionId, - request.PayloadId, - "", - parentTaskId.Equals(sessionId) - ? "" - : parentTaskId, - parentTaskId, - ResultStatus.Completed, - new List(), - DateTime.UtcNow, - size, - Array.Empty())) + r) => new Result(sessionId, + request.PayloadId, + "", + parentTaskId.Equals(sessionId) + ? "" + : parentTaskId, + parentTaskId, + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + r.size, + r.id)) .AsICollection(), cancellationToken) .ConfigureAwait(false); @@ -513,12 +523,20 @@ await taskTable_.SetTaskSuccessAsync(taskDataEnd, if (submitterOptions_.DeletePayload) { //Discard value is used to remove warnings CS4014 !! - _ = Task.Factory.StartNew(async () => await objectStorage_.TryDeleteAsync(new[] - { - taskData.PayloadId, - }, - CancellationToken.None) - .ConfigureAwait(false), + _ = Task.Factory.StartNew(async () => + { + var opaqueId = (await resultTable_.GetResult(taskData.PayloadId, + CancellationToken.None) + .ConfigureAwait(false)).OpaqueId; + + + await objectStorage_.TryDeleteAsync(new[] + { + opaqueId, + }, + CancellationToken.None) + .ConfigureAwait(false); + }, cancellationToken); await resultTable_.MarkAsDeleted(taskData.PayloadId, @@ -531,6 +549,20 @@ await resultTable_.MarkAsDeleted(taskData.PayloadId, break; case OutputStatus.Error: + + await taskTable_.UpdateManyTasks(data => data.CreatedBy == taskData.TaskId, + new UpdateDefinition().Set(data => data.Status, + TaskStatus.Cancelled), + CancellationToken.None) + .ConfigureAwait(false); + + await resultTable_.UpdateManyResults(data => data.CreatedBy == taskData.TaskId, + new UpdateDefinition().Set(data => data.Status, + ResultStatus.Aborted), + CancellationToken.None) + .ConfigureAwait(false); + + // TODO FIXME: nothing will resubmit the task if there is a crash there if (resubmit && taskData.RetryOfIds.Count < taskData.Options.MaxRetries) { @@ -595,12 +627,20 @@ await taskTable_.SetTaskTimeoutAsync(taskDataEnd, .ConfigureAwait(false); //Discard value is used to remove warnings CS4014 !! - _ = Task.Factory.StartNew(async () => await objectStorage_.TryDeleteAsync(new[] - { - taskData.TaskId, - }, - CancellationToken.None) - .ConfigureAwait(false), + _ = Task.Factory.StartNew(async () => + { + var opaqueId = (await resultTable_.GetResult(taskData.PayloadId, + CancellationToken.None) + .ConfigureAwait(false)).OpaqueId; + + + await objectStorage_.TryDeleteAsync(new[] + { + opaqueId, + }, + CancellationToken.None) + .ConfigureAwait(false); + }, cancellationToken); logger_.LogInformation("Remove input payload of {task}", diff --git a/Common/tests/AdapterLoading/AdapterLoadingTest.cs b/Common/tests/AdapterLoading/AdapterLoadingTest.cs index 87e60586b..5c65d3fb0 100644 --- a/Common/tests/AdapterLoading/AdapterLoadingTest.cs +++ b/Common/tests/AdapterLoading/AdapterLoadingTest.cs @@ -20,6 +20,7 @@ using System.Collections.Generic; using System.IO; +using ArmoniK.Core.Common.DynamicLoading; using ArmoniK.Core.Common.Injection; using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Tests.Helpers; @@ -46,6 +47,12 @@ public class AdapterLoadingTest private static readonly string RabbitPath = $"{Path.DirectorySeparatorChar}Adaptors{Path.DirectorySeparatorChar}RabbitMQ{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.RabbitMQ.dll"; + private static readonly string SqsPath = + $"{Path.DirectorySeparatorChar}Adaptors{Path.DirectorySeparatorChar}SQS{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.SQS.dll"; + + private static readonly string PubSubPath = + $"{Path.DirectorySeparatorChar}Adaptors{Path.DirectorySeparatorChar}PubSub{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.PubSub.dll"; + public static IEnumerable TestCasesQueueLocation { @@ -55,6 +62,8 @@ public static IEnumerable TestCasesQueueLocation "ArmoniK.Core.Adapters.Amqp.QueueBuilder").SetArgDisplayNames("Amqp"); yield return new TestCaseData($"{SolutionRoot}{RabbitPath}", "ArmoniK.Core.Adapters.RabbitMQ.QueueBuilder").SetArgDisplayNames("RabbitMQ"); + yield return new TestCaseData($"{SolutionRoot}{SqsPath}", + "ArmoniK.Core.Adapters.SQS.QueueBuilder").SetArgDisplayNames("SQS"); } } @@ -63,12 +72,15 @@ public static void BuildServiceProvider(Dictionary config) var loggerProvider = new ConsoleForwardingLoggerProvider(); var logger = loggerProvider.CreateLogger("root"); + AppDomain.CurrentDomain.AssemblyResolve += new CollocatedAssemblyResolver(logger).AssemblyResolve; + var configuration = new ConfigurationManager(); configuration.AddInMemoryCollection(config); var serviceCollection = new ServiceCollection(); - serviceCollection.AddQueue(configuration, - logger); + serviceCollection.AddAdapter(configuration, + nameof(Components.QueueAdaptorSettings), + logger); serviceCollection.BuildServiceProvider(); } @@ -91,22 +103,27 @@ public void QueueShouldLoad(string path, { "Amqp:User", "User" }, + { + "SQS:ServiceURL", "http://test:4566" + }, }; - Assert.DoesNotThrow(() => BuildServiceProvider(config)); + BuildServiceProvider(config); } public static IEnumerable ConfInvalidOperationException { get { - yield return new TestCaseData(new Dictionary + yield return new TestCaseData(new InvalidOperationException(), + new Dictionary { { "Amqp:User", "User" }, }).SetArgDisplayNames("No components"); - yield return new TestCaseData(new Dictionary + yield return new TestCaseData(new InvalidOperationException(), + new Dictionary { { "Amqp:User", "User" @@ -115,7 +132,8 @@ public static IEnumerable ConfInvalidOperationException $"{Components.SettingSection}:{nameof(Components.QueueAdaptorSettings)}:{nameof(Components.QueueAdaptorSettings.ClassName)}", "" }, }).SetArgDisplayNames("Empty class"); - yield return new TestCaseData(new Dictionary + yield return new TestCaseData(new InvalidOperationException(), + new Dictionary { { "Amqp:User", "User" @@ -128,7 +146,8 @@ public static IEnumerable ConfInvalidOperationException "path" }, }).SetArgDisplayNames("Empty path"); - yield return new TestCaseData(new Dictionary + yield return new TestCaseData(new FileNotFoundException(), + new Dictionary { { "Amqp:User", "User" @@ -143,7 +162,8 @@ public static IEnumerable ConfInvalidOperationException }, }).SetArgDisplayNames("invalid path"); - yield return new TestCaseData(new Dictionary + yield return new TestCaseData(new InvalidOperationException(), + new Dictionary { { "Amqp:User", "User" @@ -157,13 +177,31 @@ public static IEnumerable ConfInvalidOperationException $"{SolutionRoot}{AmqpPath}" }, }).SetArgDisplayNames("Not implemented"); + + yield return new TestCaseData(new InvalidOperationException(), + new Dictionary + { + { + "PubSub:hey", "hey" + }, + { + $"{Components.SettingSection}:{nameof(Components.QueueAdaptorSettings)}:{nameof(Components.QueueAdaptorSettings.ClassName)}", + "ArmoniK.Core.Adapters.PubSub.QueueBuilder" + }, + { + $"{Components.SettingSection}:{nameof(Components.QueueAdaptorSettings)}:{nameof(Components.QueueAdaptorSettings.AdapterAbsolutePath)}", + $"{SolutionRoot}{PubSubPath}" + }, + }).SetArgDisplayNames("PubSub no credentials"); } } [Test] [TestCaseSource(nameof(ConfInvalidOperationException))] - public void InvalidConfShouldFail(Dictionary config) - => Assert.Throws(() => BuildServiceProvider(config)); + public void InvalidConfShouldFail(TEx ex, + Dictionary config) + where TEx : Exception + => Assert.Throws(() => BuildServiceProvider(config)); public static IEnumerable ConfTypeLoadException { diff --git a/Common/tests/ArmoniK.Core.Common.Tests.csproj b/Common/tests/ArmoniK.Core.Common.Tests.csproj index c7d63bfb9..ba818252b 100644 --- a/Common/tests/ArmoniK.Core.Common.Tests.csproj +++ b/Common/tests/ArmoniK.Core.Common.Tests.csproj @@ -19,13 +19,13 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + - - + + diff --git a/Common/tests/Helpers/SimpleAgent.cs b/Common/tests/Helpers/SimpleAgent.cs index 2f81975e0..23bc4e0db 100644 --- a/Common/tests/Helpers/SimpleAgent.cs +++ b/Common/tests/Helpers/SimpleAgent.cs @@ -82,9 +82,6 @@ public Task> NotifyResultData(string token, => Task.FromResult(Array.Empty() .AsICollection()); - public Task CancelChildTasks(CancellationToken cancellationToken) - => Task.CompletedTask; - public void Dispose() => GC.SuppressFinalize(this); } diff --git a/Common/tests/Helpers/SimpleObjectStorage.cs b/Common/tests/Helpers/SimpleObjectStorage.cs index 20226720c..782f269cd 100644 --- a/Common/tests/Helpers/SimpleObjectStorage.cs +++ b/Common/tests/Helpers/SimpleObjectStorage.cs @@ -22,8 +22,8 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -37,26 +37,19 @@ public Task Check(HealthCheckTag tag) public Task Init(CancellationToken cancellationToken) => Task.CompletedTask; - public Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) - => Task.FromResult((long)0); - - public IAsyncEnumerable GetValuesAsync(string key, - CancellationToken cancellationToken = default) - => new List - { - Encoding.UTF8.GetBytes(key), - }.ToAsyncEnumerable(); - - public Task TryDeleteAsync(IEnumerable keys, + public Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) => Task.CompletedTask; - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => new List + public Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + => Task.FromResult((Encoding.UTF8.GetBytes("id"), (long)0)); + + public IAsyncEnumerable GetValuesAsync(byte[] id, + CancellationToken cancellationToken = default) + => new List { - "key1", - "key2", + id, }.ToAsyncEnumerable(); } diff --git a/Common/tests/Helpers/SimpleRabbitClient.cs b/Common/tests/Helpers/SimpleRabbitClient.cs index 2c7c3abe9..021d57b82 100644 --- a/Common/tests/Helpers/SimpleRabbitClient.cs +++ b/Common/tests/Helpers/SimpleRabbitClient.cs @@ -32,7 +32,9 @@ namespace ArmoniK.Core.Common.Tests.Helpers; public class SimpleRabbitClient : IConnectionRabbit { private readonly ILoggerFactory loggerFactory_; - private IConnection? connection_; + + private IModel? channel_; + private IConnection? connection_; public SimpleRabbitClient() { @@ -40,8 +42,6 @@ public SimpleRabbitClient() loggerFactory_.AddProvider(new ConsoleForwardingLoggerProvider()); } - public IModel? Channel { get; private set; } - public void Dispose() { if (connection_ is not null && connection_.IsOpen) @@ -49,10 +49,18 @@ public void Dispose() connection_.Close(); } + if (channel_ is not null && channel_.IsOpen) + { + channel_.Close(); + } + loggerFactory_.Dispose(); GC.SuppressFinalize(this); } + public Task GetConnectionAsync(CancellationToken cancellationToken = default) + => Task.FromResult(channel_!); + public Task Init(CancellationToken cancellation) { var connectionFactory = new ConnectionFactory @@ -62,13 +70,13 @@ public Task Init(CancellationToken cancellation) }; connection_ = connectionFactory.CreateConnection(); - Channel = connection_!.CreateModel(); + channel_ = connection_!.CreateModel(); return Task.CompletedTask; } public Task Check(HealthCheckTag tag) - => Task.FromResult(connection_ is not null && connection_.IsOpen && Channel is not null && Channel.IsOpen + => Task.FromResult(connection_ is not null && connection_.IsOpen && channel_ is not null && channel_.IsOpen ? HealthCheckResult.Healthy() : HealthCheckResult.Unhealthy()); } diff --git a/Common/tests/Helpers/SimpleWorkerStreamHandler.cs b/Common/tests/Helpers/SimpleWorkerStreamHandler.cs index 328462959..cc700b07f 100644 --- a/Common/tests/Helpers/SimpleWorkerStreamHandler.cs +++ b/Common/tests/Helpers/SimpleWorkerStreamHandler.cs @@ -29,6 +29,9 @@ namespace ArmoniK.Core.Common.Tests.Helpers; public class SimpleWorkerStreamHandler : IWorkerStreamHandler { + public Output Output = new(OutputStatus.Success, + ""); + public Task Check(HealthCheckTag tag) => Task.FromResult(HealthCheckResult.Healthy()); @@ -42,6 +45,5 @@ public Task StartTaskProcessing(TaskData taskData, string token, string dataFolder, CancellationToken cancellationToken) - => Task.FromResult(new Output(OutputStatus.Success, - "")); + => Task.FromResult(Output); } diff --git a/Common/tests/Helpers/TestDatabaseProvider.cs b/Common/tests/Helpers/TestDatabaseProvider.cs index 2b0aeaab9..80dab80d6 100644 --- a/Common/tests/Helpers/TestDatabaseProvider.cs +++ b/Common/tests/Helpers/TestDatabaseProvider.cs @@ -21,8 +21,10 @@ using System.Threading; using ArmoniK.Api.Common.Options; +using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Adapters.MongoDB.Common; +using ArmoniK.Core.Base; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Injection; using ArmoniK.Core.Common.Injection.Options; @@ -110,10 +112,6 @@ public TestDatabaseProvider(Action? collectionConfigurato $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -146,6 +144,7 @@ public TestDatabaseProvider(Action? collectionConfigurato .AddSingleton() .Configure(o => o.CopyFrom(AuthenticatorOptions.DefaultNoAuth)) .AddLogging() + .AddSingleton() .AddSingleton(loggerProvider.CreateLogger("root")) .AddSingleton(ActivitySource) .AddSingleton(_ => client_); diff --git a/Common/tests/Helpers/TestPollingAgentProvider.cs b/Common/tests/Helpers/TestPollingAgentProvider.cs index f5b110d7c..ef15e8dcc 100644 --- a/Common/tests/Helpers/TestPollingAgentProvider.cs +++ b/Common/tests/Helpers/TestPollingAgentProvider.cs @@ -89,10 +89,6 @@ public TestPollingAgentProvider(IWorkerStreamHandler workerStreamHandler) $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -117,6 +113,7 @@ public TestPollingAgentProvider(IWorkerStreamHandler workerStreamHandler) .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton(workerStreamHandler); var computePlanOptions = builder.Configuration.GetRequiredValue(ComputePlane.SettingSection); diff --git a/Common/tests/Helpers/TestPollsterProvider.cs b/Common/tests/Helpers/TestPollsterProvider.cs index b08cc6cf1..1e0b41264 100644 --- a/Common/tests/Helpers/TestPollsterProvider.cs +++ b/Common/tests/Helpers/TestPollsterProvider.cs @@ -50,6 +50,8 @@ using MongoDB.Bson; using MongoDB.Driver; +using NUnit.Framework; + namespace ArmoniK.Core.Common.Tests.Helpers; public class TestPollsterProvider : IDisposable @@ -78,7 +80,9 @@ public class TestPollsterProvider : IDisposable public TestPollsterProvider(IWorkerStreamHandler workerStreamHandler, IAgentHandler agentHandler, IPullQueueStorage pullQueueStorage, - TimeSpan? graceDelay = null) + TimeSpan? graceDelay = null, + TimeSpan? acquireTimeout = null, + int maxError = 5) { graceDelay_ = graceDelay; var logger = NullLogger.Instance; @@ -111,10 +115,6 @@ public TestPollsterProvider(IWorkerStreamHandler workerStreamHandler, $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -130,6 +130,17 @@ public TestPollsterProvider(IWorkerStreamHandler workerStreamHandler, : graceDelay .ToString() }, + { + $"{Injection.Options.Pollster.SettingSection}:{nameof(Injection.Options.Pollster.TimeoutBeforeNextAcquisition)}", + acquireTimeout is null + ? TimeSpan.FromSeconds(10) + .ToString() + : acquireTimeout.ToString() + }, + { + $"{Injection.Options.Pollster.SettingSection}:{nameof(Injection.Options.Pollster.MaxErrorAllowed)}", + maxError.ToString() + }, { $"{Injection.Options.Pollster.SettingSection}:{nameof(Injection.Options.Pollster.SharedCacheFolder)}", Path.Combine(Path.GetTempPath(), @@ -167,6 +178,7 @@ public TestPollsterProvider(IWorkerStreamHandler workerStreamHandler, .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddOption(builder.Configuration, Injection.Options.Pollster.SettingSection) @@ -231,4 +243,24 @@ await Task.Delay(delay, Lifetime.StopApplication(); }); + + public void AssertFailAfterError(int nbError = 1) + { + for (var i = 0; i < nbError; i++) + { + if (ExceptionManager.Failed) + { + Assert.Fail($"ExceptionManager failed after {i} errors while it was expected to failed after {nbError}"); + } + + ExceptionManager.RecordError(null, + null, + "Dummy Error"); + } + + if (!ExceptionManager.Failed) + { + Assert.Fail($"ExceptionManager did not failed while it was expected to failed after {nbError}"); + } + } } diff --git a/Common/tests/Helpers/TestTaskHandlerProvider.cs b/Common/tests/Helpers/TestTaskHandlerProvider.cs index c9aa69dce..b33cc7543 100644 --- a/Common/tests/Helpers/TestTaskHandlerProvider.cs +++ b/Common/tests/Helpers/TestTaskHandlerProvider.cs @@ -22,6 +22,7 @@ using System.Threading; using ArmoniK.Api.Common.Options; +using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Base; using ArmoniK.Core.Common.gRPC.Services; @@ -111,10 +112,6 @@ public TestTaskHandlerProvider(IWorkerStreamHandler workerStreamHandler, $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -165,6 +162,7 @@ public TestTaskHandlerProvider(IWorkerStreamHandler workerStreamHandler, InitServices.SettingSection) .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() @@ -252,4 +250,8 @@ public void Dispose() .Wait(); GC.SuppressFinalize(this); } + + public T GetRequiredService() + where T : notnull + => app_.Services.GetRequiredService(); } diff --git a/Common/tests/Helpers/WrapperAgentHandler.cs b/Common/tests/Helpers/WrapperAgentHandler.cs new file mode 100644 index 000000000..e0cf375d5 --- /dev/null +++ b/Common/tests/Helpers/WrapperAgentHandler.cs @@ -0,0 +1,45 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System.Threading; +using System.Threading.Tasks; + +using ArmoniK.Core.Common.Pollster; +using ArmoniK.Core.Common.Storage; + +using Microsoft.Extensions.Logging; + +namespace ArmoniK.Core.Common.Tests.Helpers; + +public class WrapperAgentHandler : IAgentHandler +{ + private readonly IAgent agent_; + + public WrapperAgentHandler(IAgent agent) + => agent_ = agent; + + public Task Stop(CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task Start(string token, + ILogger logger, + SessionData sessionData, + TaskData taskData, + string folder, + CancellationToken cancellationToken) + => Task.FromResult(agent_); +} diff --git a/Common/tests/Pollster/AgentTest.cs b/Common/tests/Pollster/AgentTest.cs index 8f35e294f..4420526a9 100644 --- a/Common/tests/Pollster/AgentTest.cs +++ b/Common/tests/Pollster/AgentTest.cs @@ -406,15 +406,6 @@ await holder.Agent.NotifyResultData(holder.Token, CancellationToken.None) .ConfigureAwait(false); - var datAsyncEnumerable = holder.ObjectStorage.GetValuesAsync(ExpectedOutput1, - CancellationToken.None); - - var dataStored = await datAsyncEnumerable.SingleAsync(CancellationToken.None) - .ConfigureAwait(false); - - Assert.AreEqual(data, - dataStored); - await holder.Agent.FinalizeTaskCreation(CancellationToken.None) .ConfigureAwait(false); @@ -431,6 +422,15 @@ await holder.Agent.FinalizeTaskCreation(CancellationToken.None) Assert.AreEqual(data.Length, resultData.Size); + var datAsyncEnumerable = holder.ObjectStorage.GetValuesAsync(resultData.OpaqueId, + CancellationToken.None); + + var dataStored = await datAsyncEnumerable.SingleAsync(CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(data, + dataStored); + var dependents = await holder.ResultTable.GetDependents(holder.Session, ExpectedOutput1, CancellationToken.None) @@ -605,14 +605,33 @@ public async Task GetResourceDataShouldSucceed() { using var holder = new AgentHolder(); - await holder.ObjectStorage.AddOrUpdateAsync("ResourceData", - new List - { - Encoding.ASCII.GetBytes("Data1"), - Encoding.ASCII.GetBytes("Data2"), - }.Select(bytes => new ReadOnlyMemory(bytes)) - .ToAsyncEnumerable(), - CancellationToken.None) + var (id, size) = await holder.ObjectStorage.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + new List + { + Encoding.ASCII.GetBytes("Data1"), + Encoding.ASCII.GetBytes("Data2"), + }.Select(bytes => new ReadOnlyMemory(bytes)) + .ToAsyncEnumerable(), + CancellationToken.None) + .ConfigureAwait(false); + + await holder.ResultTable.Create(new List + { + new("SessionId", + "ResourceData", + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + size, + id), + }) .ConfigureAwait(false); await holder.Agent.GetResourceData(holder.Token, @@ -657,14 +676,6 @@ public async Task CreateResultsShouldSucceed() resultMetadata.Status); Assert.AreEqual(holder.TaskData.TaskId, resultMetadata.CreatedBy); - - var bytes = (await holder.ObjectStorage.GetValuesAsync(result.ResultId) - .ToListAsync() - .ConfigureAwait(false)).Single(); - - Assert.AreEqual(ByteString.CopyFromUtf8(result.Name) - .ToByteArray(), - bytes); } await holder.Agent.FinalizeTaskCreation(CancellationToken.None) @@ -684,6 +695,14 @@ await holder.Agent.FinalizeTaskCreation(CancellationToken.None) resultMetadata.Status); Assert.AreEqual(7, resultMetadata.Size); + + var bytes = (await holder.ObjectStorage.GetValuesAsync(resultMetadata.OpaqueId) + .ToListAsync() + .ConfigureAwait(false)).Single(); + + Assert.AreEqual(ByteString.CopyFromUtf8(result.Name) + .ToByteArray(), + bytes); } } diff --git a/Common/tests/Pollster/DataPrefetcherTest.cs b/Common/tests/Pollster/DataPrefetcherTest.cs index 492099a1f..032c5b83c 100644 --- a/Common/tests/Pollster/DataPrefetcherTest.cs +++ b/Common/tests/Pollster/DataPrefetcherTest.cs @@ -20,15 +20,20 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Linq.Expressions; +using System.Text; using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Common.Tests.Helpers; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Moq; @@ -52,13 +57,105 @@ public virtual void TearDown() private ActivitySource? activitySource_; + private class CustomGetResultTable : IResultTable + { + private readonly List resultIds_; + private readonly string sessionId_; + + + public CustomGetResultTable(string sessionId, + List resultIds) + { + sessionId_ = sessionId; + resultIds_ = resultIds; + } + + public Task Check(HealthCheckTag tag) + => throw new NotImplementedException(); + + public Task Init(CancellationToken cancellationToken) + => throw new NotImplementedException(); + + public ILogger Logger + => NullLogger.Instance; + + public Task ChangeResultOwnership(string oldTaskId, + IEnumerable requests, + CancellationToken cancellationToken) + => throw new NotImplementedException(); + + public Task Create(ICollection results, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task AddTaskDependencies(IDictionary> dependencies, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task DeleteResult(string key, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task DeleteResults(string sessionId, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public IAsyncEnumerable GetResults(Expression> filter, + Expression> convertor, + CancellationToken cancellationToken = default) + => resultIds_.Select(s => convertor.Compile() + .Invoke(new Result(sessionId_, + s, + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + 100, + Encoding.UTF8.GetBytes(s)))) + .ToAsyncEnumerable(); + + public Task<(IEnumerable results, int totalCount)> ListResultsAsync(Expression> filter, + Expression> orderField, + bool ascOrder, + int page, + int pageSize, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task SetTaskOwnership(ICollection<(string resultId, string taskId)> requests, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task UpdateOneResult(string resultId, + UpdateDefinition updates, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task UpdateManyResults(Expression> filter, + UpdateDefinition updates, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + } + [Test] public async Task EmptyPayloadAndOneDependency() { + const string sessionId = "SessionId"; + const string parentTaskId = "ParentTaskId"; + const string taskId = "TaskId"; + const string output1 = "Output1"; + const string dependency1 = "Dependency1"; + const string podId = "PodId"; + const string podName = "PodName"; + const string payloadId = "PayloadId"; + const string createdBy = "CreatedBy"; + var mockObjectStorage = new Mock(); - mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), + mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), CancellationToken.None)) - .Returns((string _, + .Returns((byte[] _, CancellationToken _) => new List { Convert.FromBase64String("1111"), @@ -66,21 +163,20 @@ public async Task EmptyPayloadAndOneDependency() Convert.FromBase64String("3333"), Convert.FromBase64String("4444"), }.ToAsyncEnumerable()); + var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, + new CustomGetResultTable(sessionId, + new List + { + dependency1, + payloadId, + }), activitySource_, loggerFactory.CreateLogger()); - const string sessionId = "SessionId"; - const string parentTaskId = "ParentTaskId"; - const string taskId = "TaskId"; - const string output1 = "Output1"; - const string dependency1 = "Dependency1"; - const string podId = "PodId"; - const string podName = "PodName"; - const string payloadId = "PayloadId"; - const string createdBy = "CreatedBy"; + var sharedFolder = Path.Combine(Path.GetTempPath(), "data"); var internalFolder = Path.Combine(Path.GetTempPath(), @@ -135,10 +231,22 @@ await dataPrefetcher.PrefetchDataAsync(new TaskData(sessionId, [Test] public async Task EmptyPayloadAndNoDependenciesStateMachine() { + const string sessionId = "SessionId"; + const string parentTaskId = "ParentTaskId"; + const string taskId = "TaskId"; + const string output1 = "Output1"; + const string dependency1 = "Dependency1"; + const string dependency2 = "Dependency2"; + const string podId = "PodId"; + const string podName = "PodName"; + const string payloadId = "PayloadId"; + const string createdBy = "CreatedBy"; + + var mockObjectStorage = new Mock(); - mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), + mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), CancellationToken.None)) - .Returns((string _, + .Returns((byte[] _, CancellationToken _) => new List { Convert.FromBase64String("1111"), @@ -150,19 +258,17 @@ public async Task EmptyPayloadAndNoDependenciesStateMachine() var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, + new CustomGetResultTable(sessionId, + new List + { + dependency1, + dependency2, + payloadId, + }), activitySource_, loggerFactory.CreateLogger()); - const string sessionId = "SessionId"; - const string parentTaskId = "ParentTaskId"; - const string taskId = "TaskId"; - const string output1 = "Output1"; - const string dependency1 = "Dependency1"; - const string dependency2 = "Dependency2"; - const string podId = "PodId"; - const string podName = "PodName"; - const string payloadId = "PayloadId"; - const string createdBy = "CreatedBy"; + var sharedFolder = Path.Combine(Path.GetTempPath(), "data"); var internalFolder = Path.Combine(Path.GetTempPath(), @@ -219,6 +325,7 @@ public async Task InitShouldSucceed() var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, + new SimpleResultTable(), activitySource_, loggerFactory.CreateLogger()); diff --git a/Common/tests/Pollster/PollsterTest.cs b/Common/tests/Pollster/PollsterTest.cs index 40bf1463b..f36b4d238 100644 --- a/Common/tests/Pollster/PollsterTest.cs +++ b/Common/tests/Pollster/PollsterTest.cs @@ -25,7 +25,7 @@ using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Storage; @@ -313,6 +313,8 @@ await testServiceProvider.Pollster.Init(CancellationToken.None) Assert.AreEqual(HealthStatus.Healthy, (await testServiceProvider.Pollster.Check(HealthCheckTag.Startup) .ConfigureAwait(false)).Status); + + testServiceProvider.AssertFailAfterError(6); } [Test] @@ -411,6 +413,8 @@ await testServiceProvider.Pollster.Init(CancellationToken.None) Assert.DoesNotThrowAsync(() => stop); Assert.AreEqual(Array.Empty(), testServiceProvider.Pollster.TaskProcessing); + + testServiceProvider.AssertFailAfterError(6); } public class WaitWorkerStreamHandler : IWorkerStreamHandler @@ -479,12 +483,119 @@ await testServiceProvider.Pollster.Init(CancellationToken.None) Assert.DoesNotThrowAsync(() => testServiceProvider.Pollster.MainLoop()); Assert.DoesNotThrowAsync(() => stop); - Assert.False(testServiceProvider.ExceptionManager.Failed); Assert.AreEqual(TaskStatus.Completed, await testServiceProvider.TaskTable.GetTaskStatus(taskSubmitted, CancellationToken.None) .ConfigureAwait(false)); + + testServiceProvider.AssertFailAfterError(6); + } + + [Test] + [Timeout(10000)] + public async Task ExecuteTaskTimeoutAcquire() + { + var mockPullQueueStorage = new SimplePullQueueStorageChannel(); + var waitWorkerStreamHandler = new WaitWorkerStreamHandler(1000); + var simpleAgentHandler = new SimpleAgentHandler(); + + using var testServiceProvider = new TestPollsterProvider(waitWorkerStreamHandler, + simpleAgentHandler, + mockPullQueueStorage, + TimeSpan.FromMilliseconds(100), + TimeSpan.FromMilliseconds(100), + 0); + + var (sessionId, _, taskSubmitted) = await InitSubmitter(testServiceProvider.Submitter, + testServiceProvider.PartitionTable, + testServiceProvider.ResultTable, + testServiceProvider.SessionTable, + CancellationToken.None) + .ConfigureAwait(false); + + await mockPullQueueStorage.Channel.Writer.WriteAsync(new SimpleQueueMessageHandler + { + CancellationToken = CancellationToken.None, + Status = QueueMessageStatus.Waiting, + MessageId = Guid.NewGuid() + .ToString(), + TaskId = taskSubmitted, + }) + .ConfigureAwait(false); + + var expectedOutput3 = "ExpectedOutput3"; + await testServiceProvider.ResultTable.Create(new[] + { + new Result(sessionId, + expectedOutput3, + "", + "", + "", + ResultStatus.Created, + new List(), + DateTime.UtcNow, + 0, + Array.Empty()), + }, + CancellationToken.None) + .ConfigureAwait(false); + + var requests = await testServiceProvider.Submitter.CreateTasks(sessionId, + sessionId, + new TaskOptions(), + new List + { + new(new[] + { + expectedOutput3, + }, + new List(), + new List> + { + new(Encoding.ASCII.GetBytes("AAAA")), + }.ToAsyncEnumerable()), + }.ToAsyncEnumerable(), + CancellationToken.None) + .ConfigureAwait(false); + + var sessionData = await testServiceProvider.SessionTable.GetSessionAsync(sessionId, + CancellationToken.None) + .ConfigureAwait(false); + + await testServiceProvider.Submitter.FinalizeTaskCreation(requests, + sessionData, + sessionId, + CancellationToken.None) + .ConfigureAwait(false); + + var taskSubmitted2 = requests.First() + .TaskId; + + await mockPullQueueStorage.Channel.Writer.WriteAsync(new SimpleQueueMessageHandler + { + CancellationToken = CancellationToken.None, + Status = QueueMessageStatus.Waiting, + MessageId = Guid.NewGuid() + .ToString(), + TaskId = taskSubmitted2, + }) + .ConfigureAwait(false); + + await testServiceProvider.Pollster.Init(CancellationToken.None) + .ConfigureAwait(false); + + var stop = testServiceProvider.StopApplicationAfter(TimeSpan.FromSeconds(2)); + + Assert.DoesNotThrowAsync(() => testServiceProvider.Pollster.MainLoop()); + Assert.DoesNotThrowAsync(() => stop); + + Assert.AreEqual(TaskStatus.Submitted, + await testServiceProvider.TaskTable.GetTaskStatus(taskSubmitted2, + CancellationToken.None) + .ConfigureAwait(false)); + + testServiceProvider.AssertFailAfterError(); } [Test] @@ -535,6 +646,8 @@ await Task.Delay(TimeSpan.FromMilliseconds(200), await testServiceProvider.TaskTable.GetTaskStatus(taskSubmitted, CancellationToken.None) .ConfigureAwait(false)); + + testServiceProvider.AssertFailAfterError(5); } [Test] @@ -593,7 +706,6 @@ await testServiceProvider.Pollster.StopCancelledTask() Assert.DoesNotThrowAsync(() => mainLoopTask); Assert.DoesNotThrowAsync(() => stop); - Assert.False(testServiceProvider.ExceptionManager.Failed); Assert.That(await testServiceProvider.TaskTable.GetTaskStatus(taskSubmitted, CancellationToken.None) @@ -603,6 +715,8 @@ await testServiceProvider.Pollster.StopCancelledTask() Assert.AreEqual(Array.Empty(), testServiceProvider.Pollster.TaskProcessing); + + testServiceProvider.AssertFailAfterError(5); } public static IEnumerable ExecuteTooManyErrorShouldFailTestCase @@ -733,5 +847,7 @@ await testServiceProvider.TaskTable.GetTaskStatus(taskSubmitted, .ConfigureAwait(false)); Assert.AreEqual(Array.Empty(), testServiceProvider.Pollster.TaskProcessing); + + testServiceProvider.AssertFailAfterError(5); } } diff --git a/Common/tests/Pollster/TaskHandlerTest.cs b/Common/tests/Pollster/TaskHandlerTest.cs index 45b197d79..093f308d0 100644 --- a/Common/tests/Pollster/TaskHandlerTest.cs +++ b/Common/tests/Pollster/TaskHandlerTest.cs @@ -18,21 +18,27 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; using System.Linq; using System.Linq.Expressions; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; +using ArmoniK.Core.Common.Meter; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Pollster.TaskProcessingChecker; using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Common.Stream.Worker; using ArmoniK.Core.Common.Tests.Helpers; +using ArmoniK.Core.Common.Utils; using Grpc.Core; @@ -410,6 +416,16 @@ await testServiceProvider.ResultTable.Create(results) .ResultId, }, new List()), + new("TaskRetry2+Pending", + results[0] + .ResultId, + options, + new List + { + results[1] + .ResultId, + }, + new List()), }; await TaskLifeCycleHelper.CreateTasks(testServiceProvider.TaskTable, @@ -504,6 +520,23 @@ await testServiceProvider.TaskTable.SetTaskRetryAsync(taskData, await testServiceProvider.TaskTable.SetTaskRetryAsync(taskData, "Error for test : not found") .ConfigureAwait(false); + + + taskData = await testServiceProvider.TaskTable.ReadTaskAsync("TaskRetry2+Pending") + .ConfigureAwait(false); + + await testServiceProvider.TaskTable.SetTaskRetryAsync(taskData, + "Error for test : pending") + .ConfigureAwait(false); + + newTaskId = await testServiceProvider.TaskTable.RetryTask(taskData) + .ConfigureAwait(false); + + await testServiceProvider.TaskTable.UpdateOneTask(newTaskId, + null, + new UpdateDefinition().Set(data => data.Status, + TaskStatus.Pending)) + .ConfigureAwait(false); } [Test] @@ -945,6 +978,9 @@ public static IEnumerable TestCaseAcquireRetriedTask yield return new TestCaseData("TaskRetry2+NotFound").Returns(new AcquireTaskReturn(AcquisitionStatus.TaskIsRetriedAndRetryIsNotFound, TaskStatus.Retried, QueueMessageStatus.Poisonous)); + yield return new TestCaseData("TaskRetry2+Pending").Returns(new AcquireTaskReturn(AcquisitionStatus.TaskIsRetriedAndRetryIsPending, + TaskStatus.Retried, + QueueMessageStatus.Poisonous)); } } @@ -1582,6 +1618,165 @@ await testServiceProvider.TaskHandler.PostProcessing() sqmh.Status); } + [Test] + public async Task ExecuteErrorTaskAndAbortChildrenShouldSucceed() + { + var sqmh = new SimpleQueueMessageHandler + { + CancellationToken = CancellationToken.None, + Status = QueueMessageStatus.Waiting, + MessageId = Guid.NewGuid() + .ToString(), + }; + + var sh = new SimpleWorkerStreamHandler + { + Output = new Output(OutputStatus.Error, + "Error task to validate child tasks are cancelled properly"), + }; + using var testServiceProvider = new TestTaskHandlerProvider(sh, + new SimpleAgentHandler(), + sqmh); + + var (taskId, _, _, _, sessionId) = await InitProviderRunnableTask(testServiceProvider) + .ConfigureAwait(false); + + + var sessionData = await testServiceProvider.SessionTable.GetSessionAsync(sessionId) + .ConfigureAwait(false); + var taskData = await testServiceProvider.TaskTable.ReadTaskAsync(taskId, + CancellationToken.None) + .ConfigureAwait(false); + + var token = Guid.NewGuid() + .ToString(); + + var agent = new Agent(testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService(), + sessionData, + taskData, + Path.GetTempFileName(), + token, + testServiceProvider.Logger); + + var payloadId = (await agent.CreateResults(token, + new[] + { + (new ResultCreationRequest(sessionId, + "payload"), new ReadOnlyMemory(Encoding.UTF8.GetBytes("payload"))), + }, + CancellationToken.None) + .ConfigureAwait(false)).Single() + .ResultId; + + var output = (await agent.CreateResultsMetaData(token, + new[] + { + new ResultCreationRequest(sessionId, + "output"), + }, + CancellationToken.None) + .ConfigureAwait(false)).Single() + .ResultId; + + var task = (await agent.SubmitTasks(new List + { + new(payloadId, + null, + new List + { + output, + }, + new List()), + }, + null, + sessionId, + token, + CancellationToken.None) + .ConfigureAwait(false)).Single() + .TaskId; + + var agentHandler = new WrapperAgentHandler(agent); + + var taskHandler = new TaskHandler(testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService(), + sh, + sqmh, + testServiceProvider.GetRequiredService(), + "ownerpodid", + "ownerpodname", + testServiceProvider.GetRequiredService(), + agentHandler, + testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService(), + () => + { + }, + testServiceProvider.GetRequiredService(), + testServiceProvider.GetRequiredService>()); + + sqmh.TaskId = taskId; + + var acquired = await taskHandler.AcquireTask() + .ConfigureAwait(false); + + Assert.AreEqual(AcquisitionStatus.Acquired, + acquired); + + await taskHandler.PreProcessing() + .ConfigureAwait(false); + + await taskHandler.ExecuteTask() + .ConfigureAwait(false); + + await taskHandler.PostProcessing() + .ConfigureAwait(false); + + taskData = await testServiceProvider.TaskTable.ReadTaskAsync(taskId, + CancellationToken.None) + .ConfigureAwait(false); + + Console.WriteLine(taskData); + + Assert.AreEqual(TaskStatus.Error, + taskData.Status); + Assert.IsNotNull(taskData.StartDate); + Assert.IsNotNull(taskData.EndDate); + Assert.IsNotNull(taskData.ProcessingToEndDuration); + Assert.IsNotNull(taskData.CreationToEndDuration); + Assert.Greater(taskData.CreationToEndDuration, + taskData.ProcessingToEndDuration); + + Assert.AreEqual(QueueMessageStatus.Processed, + sqmh.Status); + + taskData = await testServiceProvider.TaskTable.ReadTaskAsync(task, + CancellationToken.None) + .ConfigureAwait(false); + Console.WriteLine(taskData); + Assert.AreEqual(TaskStatus.Cancelled, + taskData.Status); + + var result = await testServiceProvider.ResultTable.GetResult(payloadId) + .ConfigureAwait(false); + + Assert.AreEqual(ResultStatus.Aborted, + result.Status); + + result = await testServiceProvider.ResultTable.GetResult(output) + .ConfigureAwait(false); + + Assert.AreEqual(ResultStatus.Aborted, + result.Status); + } + + private class ObjectStorageThrowNotFound : IObjectStorage { public Task Check(HealthCheckTag tag) @@ -1590,21 +1785,18 @@ public Task Check(HealthCheckTag tag) public Task Init(CancellationToken cancellationToken) => Task.CompletedTask; - public Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) - => Task.FromResult(42); - - public IAsyncEnumerable GetValuesAsync(string key, + public IAsyncEnumerable GetValuesAsync(byte[] id, CancellationToken cancellationToken = default) => throw new ObjectDataNotFoundException(); - public Task TryDeleteAsync(IEnumerable keys, + public Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) => Task.CompletedTask; - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); + public Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + => Task.FromResult<(byte[] id, long size)>((Encoding.UTF8.GetBytes("forty-two"), 42)); } [Test] diff --git a/Common/tests/Submitter/GrpcSubmitterServiceTests.cs b/Common/tests/Submitter/GrpcSubmitterServiceTests.cs index 312fe17ad..ccc8ff0b9 100644 --- a/Common/tests/Submitter/GrpcSubmitterServiceTests.cs +++ b/Common/tests/Submitter/GrpcSubmitterServiceTests.cs @@ -25,6 +25,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; diff --git a/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs b/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs index c1f0f4bc6..0e04b6ad4 100644 --- a/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs +++ b/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; diff --git a/Common/tests/Submitter/SubmitterTests.cs b/Common/tests/Submitter/SubmitterTests.cs index 826741392..756f0c103 100644 --- a/Common/tests/Submitter/SubmitterTests.cs +++ b/Common/tests/Submitter/SubmitterTests.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.Common.Options; using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Base; using ArmoniK.Core.Common.Exceptions; @@ -96,10 +97,6 @@ public async Task SetUp() $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -128,6 +125,7 @@ public async Task SetUp() .AddSingleton(client_) .AddLogging(builder => builder.AddProvider(loggerProvider)) .AddSingleton() + .AddSingleton() .AddOption(configuration, Injection.Options.Submitter.SettingSection) .AddInitializedOption(configuration, @@ -974,10 +972,7 @@ await submitter_.TryGetResult(resultRequest, Assert.AreEqual(ResultReply.TypeOneofCase.Result, writer.Messages[1] .TypeCase); - Assert.AreEqual(ResultReply.TypeOneofCase.Result, - writer.Messages[2] - .TypeCase); - Assert.IsTrue(writer.Messages[2] + Assert.IsTrue(writer.Messages[1] .Result.DataComplete); } diff --git a/Common/tests/TaskLifeCycleHelperTest.cs b/Common/tests/TaskLifeCycleHelperTest.cs index 04421b3f5..6535c0fa7 100644 --- a/Common/tests/TaskLifeCycleHelperTest.cs +++ b/Common/tests/TaskLifeCycleHelperTest.cs @@ -427,7 +427,8 @@ await TaskLifeCycleHelper.FinalizeTaskCreation(holder.TaskTable, await holder.ResultTable.CompleteResult(sessionId, results[1] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("first data dependency")) .ConfigureAwait(false); await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, @@ -455,7 +456,8 @@ await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, await holder.ResultTable.CompleteResult(sessionId, results[2] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("second data dependency")) .ConfigureAwait(false); await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, @@ -613,12 +615,14 @@ await TaskLifeCycleHelper.FinalizeTaskCreation(holder.TaskTable, await holder.ResultTable.CompleteResult(sessionId, results[1] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("first data dependency")) .ConfigureAwait(false); await holder.ResultTable.CompleteResult(sessionId, results[2] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("second data dependency")) .ConfigureAwait(false); await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, holder.ResultTable, diff --git a/Common/tests/TestBase/ObjectStorageTestBase.cs b/Common/tests/TestBase/ObjectStorageTestBase.cs index 72248c1b4..ef1b37cd3 100644 --- a/Common/tests/TestBase/ObjectStorageTestBase.cs +++ b/Common/tests/TestBase/ObjectStorageTestBase.cs @@ -22,9 +22,9 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base.Exceptions; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -55,25 +55,37 @@ public async Task SetUp() Encoding.ASCII.GetBytes("CCCC"), Encoding.ASCII.GetBytes("DDDD"), }; - await ObjectStorage!.AddOrUpdateAsync("dataKey1", - dataBytesList.ToAsyncEnumerable()) - .ConfigureAwait(false); + datakey1_ = (await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + dataBytesList.ToAsyncEnumerable()) + .ConfigureAwait(false)).id; dataBytesList = new List> { Encoding.ASCII.GetBytes("AAAABBBB"), }; - await ObjectStorage.AddOrUpdateAsync("dataKey2", - dataBytesList.ToAsyncEnumerable()) - .ConfigureAwait(false); + datakey2_ = (await ObjectStorage.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + dataBytesList.ToAsyncEnumerable()) + .ConfigureAwait(false)).id; dataBytesList = new List> { Array.Empty(), }; - await ObjectStorage.AddOrUpdateAsync("dataKeyEmpty", - dataBytesList.ToAsyncEnumerable()) - .ConfigureAwait(false); + datakeyEmpty_ = (await ObjectStorage.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + dataBytesList.ToAsyncEnumerable()) + .ConfigureAwait(false)).id; } [TearDown] @@ -94,7 +106,10 @@ private static bool CheckForSkipSetup() /* Boolean to control that tests are executed in * an instance of this class */ - protected bool RunTests; + protected bool RunTests; + private byte[]? datakey1_; + private byte[]? datakey2_; + private byte[]? datakeyEmpty_; /* Function be override so it returns the suitable instance * of TaskTable to the corresponding interface implementation */ @@ -138,11 +153,15 @@ public async Task AddValuesAsyncWithoutChunkShouldWork() { if (RunTests) { - var size = await ObjectStorage!.AddOrUpdateAsync("dataKeyNoChunk", - AsyncEnumerable.Empty>()) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + AsyncEnumerable.Empty>()) + .ConfigureAwait(false); var data = new List(); - await foreach (var chunk in ObjectStorage!.GetValuesAsync("dataKeyNoChunk") + await foreach (var chunk in ObjectStorage!.GetValuesAsync(id) .ConfigureAwait(false)) { data.AddRange(chunk); @@ -174,13 +193,17 @@ public async Task AddValuesAsyncShouldWork(params string[] inputChunks) { if (RunTests) { - var size = await ObjectStorage!.AddOrUpdateAsync("dataKeyTest", - inputChunks.ToAsyncEnumerable() - .Select(s => (ReadOnlyMemory)Encoding.ASCII.GetBytes(s))) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + inputChunks.ToAsyncEnumerable() + .Select(s => (ReadOnlyMemory)Encoding.ASCII.GetBytes(s))) + .ConfigureAwait(false); var data = new List(); - await foreach (var chunk in ObjectStorage!.GetValuesAsync("dataKeyTest") + await foreach (var chunk in ObjectStorage!.GetValuesAsync(id) .ConfigureAwait(false)) { data.AddRange(chunk); @@ -197,13 +220,33 @@ public async Task AddValuesAsyncShouldWork(params string[] inputChunks) } [Test] - public void GetValuesAsyncShouldFail() + public async Task GetValuesAsyncShouldFail() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync("dataKeyNotExist"); - Assert.ThrowsAsync(async () => await res.FirstAsync() - .ConfigureAwait(false)); + var id = Encoding.UTF8.GetBytes("IdThatShouldFail"); + var data = new List(); + + try + { + var res = ObjectStorage!.GetValuesAsync(id); + + + await foreach (var chunk in res.ConfigureAwait(false)) + { + data.AddRange(chunk); + } + + Assert.AreEqual(id, + data.ToArray()); + } + catch (ObjectDataNotFoundException) + { + } + catch (Exception) + { + Assert.Fail("When value should not be available, it should either throw or return the value"); + } } } @@ -212,7 +255,7 @@ public async Task PayloadShouldBeEqual() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync("dataKey2"); + var res = ObjectStorage!.GetValuesAsync(datakey2_!); var data = new List(); foreach (var item in await res.ToListAsync() .ConfigureAwait(false)) @@ -222,7 +265,8 @@ public async Task PayloadShouldBeEqual() var str = Encoding.ASCII.GetString(data.ToArray()); Console.WriteLine(str); - Assert.IsTrue(str.SequenceEqual("AAAABBBB")); + Assert.AreEqual("AAAABBBB", + str); } } @@ -231,7 +275,7 @@ public async Task Payload2ShouldBeEqual() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync("dataKey1"); + var res = ObjectStorage!.GetValuesAsync(datakey1_!); // var data = await res.AggregateAsync((bytes1, bytes2) => bytes1.Concat(bytes2).ToArray()); var data = new List(); foreach (var item in await res.ToListAsync() @@ -242,7 +286,8 @@ public async Task Payload2ShouldBeEqual() var str = Encoding.ASCII.GetString(data.ToArray()); Console.WriteLine(str); - Assert.IsTrue(str.SequenceEqual("AAAABBBBCCCCDDDD")); + Assert.AreEqual("AAAABBBBCCCCDDDD", + str); } } @@ -251,7 +296,7 @@ public async Task EmptyPayload() { if (RunTests) { - var res = await ObjectStorage!.GetValuesAsync("dataKeyEmpty") + var res = await ObjectStorage!.GetValuesAsync(datakeyEmpty_!) .ToListAsync() .ConfigureAwait(false); Console.WriteLine(res.Count); @@ -267,44 +312,6 @@ public async Task EmptyPayload() } } - [Test] - public async Task DeleteKeysAndGetValuesAsyncShouldFail() - { - if (RunTests) - { - var listChunks = new List> - { - Encoding.ASCII.GetBytes("Armonik Payload chunk"), - Encoding.ASCII.GetBytes("Data 1"), - Encoding.ASCII.GetBytes("Data 2"), - Encoding.ASCII.GetBytes("Data 3"), - Encoding.ASCII.GetBytes("Data 4"), - }; - - await ObjectStorage!.AddOrUpdateAsync("dataKey", - listChunks.ToAsyncEnumerable()) - .ConfigureAwait(false); - - var res = await ObjectStorage!.GetValuesAsync("dataKey") - .ToListAsync() - .ConfigureAwait(false); - Assert.AreEqual(string.Join("", - listChunks.Select(chunk => Encoding.ASCII.GetString(chunk.ToArray()))), - string.Join("", - res.Select(chunk => Encoding.ASCII.GetString(chunk)))); - - await ObjectStorage!.TryDeleteAsync(new[] - { - "dataKey", - }) - .ConfigureAwait(false); - - Assert.ThrowsAsync(async () => await ObjectStorage!.GetValuesAsync("dataKey") - .FirstAsync() - .ConfigureAwait(false)); - } - } - [Test] public async Task DeleteDeleteTwiceShouldSucceed() { @@ -319,19 +326,23 @@ public async Task DeleteDeleteTwiceShouldSucceed() Encoding.ASCII.GetBytes("Data 4"), }; - await ObjectStorage!.AddOrUpdateAsync("dataKey", - listChunks.ToAsyncEnumerable()) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + listChunks.ToAsyncEnumerable()) + .ConfigureAwait(false); await ObjectStorage!.TryDeleteAsync(new[] { - "dataKey", + id, }) .ConfigureAwait(false); await ObjectStorage!.TryDeleteAsync(new[] { - "dataKey", + id, }) .ConfigureAwait(false); } diff --git a/Common/tests/TestBase/ResultTableTestBase.cs b/Common/tests/TestBase/ResultTableTestBase.cs index 797fd1976..1c39e1068 100644 --- a/Common/tests/TestBase/ResultTableTestBase.cs +++ b/Common/tests/TestBase/ResultTableTestBase.cs @@ -23,6 +23,7 @@ using System.Threading.Tasks; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; @@ -323,48 +324,25 @@ public async Task SetResultShouldSucceed() { if (RunTests) { - await ResultTable!.SetResult((string)"SessionId", - (string)"OwnerId", - (string)"ResultIsNotAvailable", - 5, - CancellationToken.None) - .ConfigureAwait(false); - - var result = await ResultTable!.GetResult("ResultIsNotAvailable", - CancellationToken.None) - .ConfigureAwait(false); - - Assert.IsTrue(result.ResultId == "ResultIsNotAvailable"); - Assert.AreEqual(5, - result.Size); - } - } - - [Test] - public async Task SetResultSmallPayloadShouldSucceed() - { - if (RunTests) - { - var smallPayload = new[] - { - (byte)1, - (byte)2, - }; - + var id = Encoding.UTF8.GetBytes("OpaqueId"); await ResultTable!.SetResult("SessionId", "OwnerId", "ResultIsNotAvailable", - smallPayload, + 5, + id, CancellationToken.None) .ConfigureAwait(false); + var result = await ResultTable!.GetResult("ResultIsNotAvailable", CancellationToken.None) .ConfigureAwait(false); - Assert.AreEqual(result.Data, - smallPayload); - Assert.AreEqual(smallPayload.Length, + Assert.AreEqual("ResultIsNotAvailable", + result.ResultId); + Assert.AreEqual(5, result.Size); + Assert.AreEqual(id, + result.OpaqueId); } } @@ -769,6 +747,7 @@ public async Task CompleteResultShouldSucceed() .ToString(); var sessionId = Guid.NewGuid() .ToString(); + var id = Encoding.UTF8.GetBytes("OpaqueId"); await ResultTable!.Create(new List { new(sessionId, @@ -788,11 +767,16 @@ public async Task CompleteResultShouldSucceed() var result = await ResultTable.CompleteResult(sessionId, resultId, 5, + id, CancellationToken.None) .ConfigureAwait(false); Assert.AreEqual(ResultStatus.Completed, result.Status); + Assert.AreEqual(id, + result.OpaqueId); + Assert.AreEqual(5, + result.Size); result = await ResultTable.GetResult(resultId, CancellationToken.None) @@ -802,6 +786,8 @@ public async Task CompleteResultShouldSucceed() result.Status); Assert.AreEqual(5, result.Size); + Assert.AreEqual(id, + result.OpaqueId); } } @@ -810,9 +796,11 @@ public void CompleteResultShouldThrow() { if (RunTests) { + var id = Encoding.UTF8.GetBytes("OpaqueId"); Assert.ThrowsAsync(async () => await ResultTable!.CompleteResult("SessionId", "NotExistingResult111", 5, + id, CancellationToken.None) .ConfigureAwait(false)); } diff --git a/Common/tests/TestBase/TaskTableTestBase.cs b/Common/tests/TestBase/TaskTableTestBase.cs index 75749cfd8..c84fa8030 100644 --- a/Common/tests/TestBase/TaskTableTestBase.cs +++ b/Common/tests/TestBase/TaskTableTestBase.cs @@ -32,6 +32,7 @@ using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Api.gRPC.V1.Tasks; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC; using ArmoniK.Core.Common.gRPC.Convertors; @@ -2151,8 +2152,8 @@ public async Task RemoveRemainingDataDependenciesShouldSucceed() { var taskId = Guid.NewGuid() .ToString(); - var dd1 = "dependency.1"; - var dd2 = "dependency.2"; + var dd1 = "dependency1"; + var dd2 = "dependency2"; await TaskTable!.CreateTasks(new[] { diff --git a/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj b/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj index de55cc27f..ccc01cead 100644 --- a/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj +++ b/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj @@ -36,10 +36,7 @@ - - - diff --git a/Compute/PollingAgent/src/Program.cs b/Compute/PollingAgent/src/Program.cs index cad0f4989..9d56bfbb2 100644 --- a/Compute/PollingAgent/src/Program.cs +++ b/Compute/PollingAgent/src/Program.cs @@ -22,12 +22,10 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Core.Adapters.LocalStorage; using ArmoniK.Core.Adapters.MongoDB; -using ArmoniK.Core.Adapters.Redis; -using ArmoniK.Core.Adapters.S3; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Common.DynamicLoading; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Injection; using ArmoniK.Core.Common.Injection.Options; @@ -82,6 +80,8 @@ public static async Task Main(string[] args) try { + AppDomain.CurrentDomain.AssemblyResolve += new CollocatedAssemblyResolver(logger.GetLogger()).AssemblyResolve; + var pollsterOptions = builder.Configuration.GetSection(Pollster.SettingSection) .Get() ?? new Pollster(); @@ -92,14 +92,12 @@ public static async Task Main(string[] args) .AddArmoniKWorkerConnection(builder.Configuration) .AddMongoComponents(builder.Configuration, logger.GetLogger()) - .AddQueue(builder.Configuration, - logger.GetLogger()) - .AddRedis(builder.Configuration, - logger.GetLogger()) - .AddS3(builder.Configuration, - logger.GetLogger()) - .AddLocalStorage(builder.Configuration, - logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.QueueAdaptorSettings), + logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()) .AddHostedService() .AddHostedService() .AddHostedService() diff --git a/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj b/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj index 10d4781b6..53290c993 100644 --- a/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj +++ b/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj @@ -35,10 +35,7 @@ - - - diff --git a/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj b/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj index 6610aa411..5a70d5b96 100644 --- a/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj +++ b/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj @@ -37,10 +37,7 @@ - - - diff --git a/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj b/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj index 9177894ff..72201f23b 100644 --- a/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj +++ b/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj @@ -38,9 +38,6 @@ - - - diff --git a/Control/Submitter/src/Program.cs b/Control/Submitter/src/Program.cs index 5af0b7e6f..27f4377a0 100644 --- a/Control/Submitter/src/Program.cs +++ b/Control/Submitter/src/Program.cs @@ -22,13 +22,11 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Core.Adapters.LocalStorage; using ArmoniK.Core.Adapters.MongoDB; -using ArmoniK.Core.Adapters.Redis; -using ArmoniK.Core.Adapters.S3; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Common.Auth.Authentication; +using ArmoniK.Core.Common.DynamicLoading; using ArmoniK.Core.Common.gRPC; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Injection; @@ -84,20 +82,20 @@ public static async Task Main(string[] args) try { + AppDomain.CurrentDomain.AssemblyResolve += new CollocatedAssemblyResolver(logger.GetLogger()).AssemblyResolve; + builder.Host.UseSerilog(logger.GetSerilogConf()); builder.Services.AddLogging(logger.Configure) .AddHttpClient() .AddMongoComponents(builder.Configuration, logger.GetLogger()) - .AddQueue(builder.Configuration, - logger.GetLogger()) - .AddRedis(builder.Configuration, - logger.GetLogger()) - .AddLocalStorage(builder.Configuration, - logger.GetLogger()) - .AddS3(builder.Configuration, - logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.QueueAdaptorSettings), + logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()) .AddSingleton() .AddSingletonWithHealthCheck(nameof(ExceptionInterceptor)) .AddOption(builder.Configuration, diff --git a/Control/Submitter/tests/ArmoniK.Core.Control.Submitter.Tests.csproj b/Control/Submitter/tests/ArmoniK.Core.Control.Submitter.Tests.csproj index 5f51f0560..8c2f4890c 100644 --- a/Control/Submitter/tests/ArmoniK.Core.Control.Submitter.Tests.csproj +++ b/Control/Submitter/tests/ArmoniK.Core.Control.Submitter.Tests.csproj @@ -19,7 +19,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Dockerfile b/Dockerfile index f8918d636..780b0e63f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ COPY ["Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj", "Adaptors/PubSu COPY ["Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj", "Adaptors/Redis/src/"] COPY ["Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj", "Adaptors/S3/src/"] COPY ["Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj", "Adaptors/SQS/src/"] +COPY ["Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj", "Adaptors/Embed/src/"] COPY ["Base/src/ArmoniK.Core.Base.csproj", "Base/src/"] COPY ["Common/src/ArmoniK.Core.Common.csproj", "Common/src/"] COPY ["Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj", "Compute/PollingAgent/src/"] @@ -40,6 +41,10 @@ RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Amqp/src/ArmoniK.Core.Adapters.A RUN dotnet restore -a "${TARGETARCH}" "Adaptors/RabbitMQ/src/ArmoniK.Core.Adapters.RabbitMQ.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj" # git ls-tree -r HEAD --name-only --full-tree | grep "csproj$" | xargs -I % sh -c "export D=\$(dirname %) ; echo COPY [\\\"\$D\\\", \\\"\$D\\\"]" COPY ["Adaptors/Amqp/src", "Adaptors/Amqp/src"] @@ -52,6 +57,7 @@ COPY ["Adaptors/PubSub/src", "Adaptors/PubSub/src"] COPY ["Adaptors/Redis/src", "Adaptors/Redis/src"] COPY ["Adaptors/S3/src", "Adaptors/S3/src"] COPY ["Adaptors/SQS/src", "Adaptors/SQS/src"] +COPY ["Adaptors/Embed/src", "Adaptors/Embed/src"] COPY ["Base/src", "Base/src"] COPY ["Common/src", "Common/src"] COPY ["Compute/PollingAgent/src", "Compute/PollingAgent/src"] @@ -72,6 +78,18 @@ RUN dotnet publish "ArmoniK.Core.Adapters.Amqp.csproj" -a "${TARGETARCH}" --no-r WORKDIR /src/Adaptors/RabbitMQ/src RUN dotnet publish "ArmoniK.Core.Adapters.RabbitMQ.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/rabbit /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION +WORKDIR /src/Adaptors/LocalStorage/src +RUN dotnet publish "ArmoniK.Core.Adapters.LocalStorage.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/local_storage /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + +WORKDIR /src/Adaptors/Embed/src +RUN dotnet publish "ArmoniK.Core.Adapters.Embed.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/embed /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + +WORKDIR /src/Adaptors/Redis/src +RUN dotnet publish "ArmoniK.Core.Adapters.Redis.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/redis /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + +WORKDIR /src/Adaptors/S3/src +RUN dotnet publish "ArmoniK.Core.Adapters.S3.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/s3 /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + WORKDIR /src/Compute/PollingAgent/src RUN dotnet publish "ArmoniK.Core.Compute.PollingAgent.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/polling_agent /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION @@ -94,6 +112,14 @@ WORKDIR /adapters/queue/amqp COPY --from=build /app/publish/amqp . WORKDIR /adapters/queue/rabbit COPY --from=build /app/publish/rabbit . +WORKDIR /adapters/object/local_storage +COPY --from=build /app/publish/local_storage . +WORKDIR /adapters/object/redis +COPY --from=build /app/publish/redis . +WORKDIR /adapters/object/embed +COPY --from=build /app/publish/embed . +WORKDIR /adapters/object/s3 +COPY --from=build /app/publish/s3 . WORKDIR /app COPY --from=build /app/publish/polling_agent . ENV ASPNETCORE_URLS http://+:1080 @@ -126,6 +152,14 @@ WORKDIR /adapters/queue/amqp COPY --from=build /app/publish/amqp . WORKDIR /adapters/queue/rabbit COPY --from=build /app/publish/rabbit . +WORKDIR /adapters/object/local_storage +COPY --from=build /app/publish/local_storage . +WORKDIR /adapters/object/redis +COPY --from=build /app/publish/redis . +WORKDIR /adapters/object/embed +COPY --from=build /app/publish/embed . +WORKDIR /adapters/object/s3 +COPY --from=build /app/publish/s3 . WORKDIR /app COPY --from=build /app/publish/submitter . ENV ASPNETCORE_URLS http://+:1080, http://+:1081 diff --git a/Tests/Bench/Client/src/ArmoniK.Samples.Bench.Client.csproj b/Tests/Bench/Client/src/ArmoniK.Samples.Bench.Client.csproj index 76b9befa2..0acb37409 100644 --- a/Tests/Bench/Client/src/ArmoniK.Samples.Bench.Client.csproj +++ b/Tests/Bench/Client/src/ArmoniK.Samples.Bench.Client.csproj @@ -28,12 +28,12 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + - - - + + + diff --git a/Tests/Bench/Client/src/Program.cs b/Tests/Bench/Client/src/Program.cs index ae2b83ece..94694797b 100644 --- a/Tests/Bench/Client/src/Program.cs +++ b/Tests/Bench/Client/src/Program.cs @@ -611,6 +611,13 @@ await results.ParallelForEach(new ParallelTaskOptions(benchOptions.DegreeOfParal var countFinished = Stopwatch.GetTimestamp(); + await channelPool.WithInstanceAsync(channel => new Sessions.SessionsClient(channel).CloseSessionAsync(new CloseSessionRequest + { + SessionId = createSessionReply.SessionId, + }), + CancellationToken.None) + .ConfigureAwait(false); + var stats = new ExecutionStats { ElapsedTime = TimeSpan.FromTicks((resultsReceived - start) / 100), diff --git a/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj b/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj index 042640bcc..8aaa0892e 100644 --- a/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj +++ b/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj b/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj index d543af582..38c0a9afd 100644 --- a/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj +++ b/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj @@ -25,7 +25,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj b/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj index 1e7515140..3a0e4c932 100644 --- a/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj +++ b/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj @@ -25,12 +25,12 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj b/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj index cf1763a8a..e563a8a68 100644 --- a/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj +++ b/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj @@ -28,13 +28,13 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + - - - - + + + + diff --git a/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj b/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj index 042640bcc..8aaa0892e 100644 --- a/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj +++ b/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/HtcMock/Client/src/ArmoniK.Samples.HtcMock.Client.csproj b/Tests/HtcMock/Client/src/ArmoniK.Samples.HtcMock.Client.csproj index b7fb7bba1..6e230917b 100644 --- a/Tests/HtcMock/Client/src/ArmoniK.Samples.HtcMock.Client.csproj +++ b/Tests/HtcMock/Client/src/ArmoniK.Samples.HtcMock.Client.csproj @@ -29,12 +29,12 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + - - - + + + diff --git a/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj b/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj index 55bd16c8d..c7e12ca67 100644 --- a/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj +++ b/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj @@ -23,7 +23,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj b/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj index 89725489b..9d98e5be4 100644 --- a/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj +++ b/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj @@ -24,12 +24,12 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj b/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj index ddc2b6b21..dc50eb0af 100644 --- a/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj +++ b/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Utils/src/ArmoniK.Core.Utils.csproj b/Utils/src/ArmoniK.Core.Utils.csproj index 9f0513ffb..20a59ebcc 100644 --- a/Utils/src/ArmoniK.Core.Utils.csproj +++ b/Utils/src/ArmoniK.Core.Utils.csproj @@ -25,17 +25,17 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + + - + diff --git a/justfile b/justfile index bb351ff71..ff17e668e 100644 --- a/justfile +++ b/justfile @@ -64,6 +64,8 @@ object_storage := if object == "redis" { '{ name = "redis", image = "redis:bullseye" ' } else if object == "minio" { '{ name = "minio", image = "quay.io/minio/minio" ' +} else if object == "embed" { + '{ name = "embed"' } else { '{ name = "local", image = "" ' } @@ -181,6 +183,7 @@ _usage: redis: to use redis for object storage (default) minio: to use minio for object storage. local: to mount a local volume for object storage + embed: to use the database as an object storage replicas: Number of polling agents / worker to be replicated (default = 3) diff --git a/terraform/locals.tf b/terraform/locals.tf index e8fc1a95f..ece102109 100644 --- a/terraform/locals.tf +++ b/terraform/locals.tf @@ -16,7 +16,7 @@ locals { worker = merge(var.compute_plane.worker, { image = var.worker_image }) queue = one(concat(module.queue_activemq, module.queue_rabbitmq, module.queue_artemis, module.queue_pubsub, module.queue_sqs, module.queue_none)) database = module.database - object = one(concat(module.object_redis, module.object_minio, module.object_local)) + object = one(concat(module.object_redis, module.object_minio, module.object_local, module.object_embed)) partition_env_vars = { for i in local.partitions : "InitServices__Partitioning__Partitions__${i}" => jsonencode(merge(var.partition_data, { PartitionId = "${var.partition_data.PartitionId}${i}" })) } env_maps = concat([ local.queue.generated_env_vars, diff --git a/terraform/main.tf b/terraform/main.tf index 467aac05e..be116ece1 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -59,6 +59,11 @@ module "object_local" { local_path = var.object_storage.local_storage_path } +module "object_embed" { + source = "./modules/storage/object/embed" + count = var.object_storage.name == "embed" ? 1 : 0 +} + module "queue_rabbitmq" { source = "./modules/storage/queue/rabbitmq" count = var.queue_storage.name == "rabbitmq" ? 1 : 0 diff --git a/terraform/modules/storage/database/mongo/versions.tf b/terraform/modules/storage/database/mongo/versions.tf index 042090bbc..b4596b999 100644 --- a/terraform/modules/storage/database/mongo/versions.tf +++ b/terraform/modules/storage/database/mongo/versions.tf @@ -6,7 +6,7 @@ terraform { } time = { source = "hashicorp/time" - version = "0.12.0" + version = "0.12.1" } } } diff --git a/terraform/modules/storage/object/embed/output.tf b/terraform/modules/storage/object/embed/output.tf new file mode 100644 index 000000000..3109defe1 --- /dev/null +++ b/terraform/modules/storage/object/embed/output.tf @@ -0,0 +1,11 @@ +output "generated_env_vars" { + value = ({ + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Embed.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/embed/ArmoniK.Core.Adapters.Embed.dll" + }) +} + +output "volumes" { + description = "Volumes that agents and submitters must mount to access the object storage" + value = {} +} diff --git a/terraform/modules/storage/object/local/output.tf b/terraform/modules/storage/object/local/output.tf index 0892e2b89..bb9ad423b 100644 --- a/terraform/modules/storage/object/local/output.tf +++ b/terraform/modules/storage/object/local/output.tf @@ -1,7 +1,8 @@ output "generated_env_vars" { value = ({ - "Components__ObjectStorage" = "ArmoniK.Adapters.LocalStorage.ObjectStorage", - "LocalStorage__Path" = var.local_path + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.LocalStorage.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/local_storage/ArmoniK.Core.Adapters.LocalStorage.dll" + "LocalStorage__Path" = var.local_path }) } diff --git a/terraform/modules/storage/object/minio/outputs.tf b/terraform/modules/storage/object/minio/outputs.tf index 6f9947d26..ad1c280ad 100644 --- a/terraform/modules/storage/object/minio/outputs.tf +++ b/terraform/modules/storage/object/minio/outputs.tf @@ -1,11 +1,12 @@ output "generated_env_vars" { value = ({ - "Components__ObjectStorage" = "ArmoniK.Adapters.S3.ObjectStorage", - "S3__EndpointUrl" = "http://${var.host}:${var.port}" - "S3__BucketName" = var.bucket_name - "S3__Login" = var.login - "S3__Password" = var.password - "S3__MustForcePathStyle" = true + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.S3.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/s3/ArmoniK.Core.Adapters.S3.dll" + "S3__EndpointUrl" = "http://${var.host}:${var.port}" + "S3__BucketName" = var.bucket_name + "S3__Login" = var.login + "S3__Password" = var.password + "S3__MustForcePathStyle" = true }) } diff --git a/terraform/modules/storage/object/redis/outputs.tf b/terraform/modules/storage/object/redis/outputs.tf index 7ffd47b7f..82b09244f 100644 --- a/terraform/modules/storage/object/redis/outputs.tf +++ b/terraform/modules/storage/object/redis/outputs.tf @@ -1,7 +1,8 @@ output "generated_env_vars" { value = ({ - "Components__ObjectStorage" = "ArmoniK.Adapters.Redis.ObjectStorage", - "Redis__EndpointUrl" = "object:${var.exposed_port}" + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Redis.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/redis/ArmoniK.Core.Adapters.Redis.dll" + "Redis__EndpointUrl" = "object:${var.exposed_port}" }) } output "volumes" { diff --git a/terraform/modules/storage/queue/sqs/outputs.tf b/terraform/modules/storage/queue/sqs/outputs.tf index 4fdfe7d4b..727b30ff8 100644 --- a/terraform/modules/storage/queue/sqs/outputs.tf +++ b/terraform/modules/storage/queue/sqs/outputs.tf @@ -4,6 +4,8 @@ output "generated_env_vars" { "Components__QueueAdaptorSettings__AdapterAbsolutePath" = "/adapters/queue/sqs/ArmoniK.Core.Adapters.SQS.dll" "SQS__ServiceURL" = "http://${var.queue_envs.host}:4566" "SQS__PartitionId" = "TestPartition0" + "SQS__Prefix" = "armonik" + "SQS__Tags__deployment" = "docker" "AWS_ACCESS_KEY_ID" = "localkey" "AWS_SECRET_ACCESS_KEY" = "localsecret" }) diff --git a/terraform/variables.tf b/terraform/variables.tf index 7dab51033..7ff41997b 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -62,8 +62,8 @@ variable "object_storage" { local_storage_path = optional(string, "/local_storage") }) validation { - condition = can(regex("^(redis|local|minio)$", var.object_storage.name)) - error_message = "Must be redis, minio, or local" + condition = can(regex("^(redis|local|minio|embed)$", var.object_storage.name)) + error_message = "Must be redis, minio, embed, or local" } default = {} }