From a64d3ef4eea4c504d8c7ba64253e849025bfad70 Mon Sep 17 00:00:00 2001 From: ahmedhanafy725 Date: Sun, 4 Feb 2024 17:15:41 +0200 Subject: [PATCH 01/42] Change the listing the deployments from the kvstore to contracts data --- .../src/clients/tf-grid/contracts.ts | 37 +++++++++ packages/grid_client/src/modules/base.ts | 83 +++++++++++++------ packages/grid_client/src/modules/zdb.ts | 2 +- .../src/components/vm_deployment_table.vue | 12 +-- 4 files changed, 100 insertions(+), 34 deletions(-) diff --git a/packages/grid_client/src/clients/tf-grid/contracts.ts b/packages/grid_client/src/clients/tf-grid/contracts.ts index 0b7a5c2749..8c2592129b 100644 --- a/packages/grid_client/src/clients/tf-grid/contracts.ts +++ b/packages/grid_client/src/clients/tf-grid/contracts.ts @@ -141,6 +141,34 @@ class TFContracts extends Contracts { } } + async listNodeContractsByTwinId(options: ListContractByTwinIdOptions): Promise { + options.stateList = options.stateList || [ContractStates.Created, ContractStates.GracePeriod]; + const state = `[${options.stateList.join(", ")}]`; + const gqlClient = new Graphql(options.graphqlURL); + const opts = `(where: {twinID_eq: ${options.twinId}, state_in: ${state}}, orderBy: twinID_ASC)`; + try { + const nodeContractsCount = await gqlClient.getItemTotalCount("nodeContracts", opts); + const body = `query getContracts($nodeContractsCount: Int!){ + nodeContracts(where: {twinID_eq: ${options.twinId}, state_in: ${state}}, limit: $nodeContractsCount) { + contractID + deploymentData + state + createdAt + nodeID + numberOfPublicIPs + } + }`; + const response = await gqlClient.query(body, { + nodeContractsCount, + }); + + return (response["data"] as GqlContracts).nodeContracts; + } catch (err) { + (err as Error).message = formatErrorMessage(`Error listing contracts by twin id ${options.twinId}.`, err); + throw err; + } + } + /** * Get contract consumption per hour in TFT. * @@ -221,6 +249,15 @@ class TFContracts extends Contracts { }); } + async listMyNodeContracts(options: ListMyContractOptions): Promise { + const twinId = await this.client.twins.getMyTwinId(); + return await this.listNodeContractsByTwinId({ + graphqlURL: options.graphqlURL, + twinId: twinId, + stateList: options.stateList, + }); + } + async contractLock(options: ContractLockOptions) { const res = await super.contractLock(options); const amountLocked = new Decimal(res.amountLocked); diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index 9ac3c7beaa..f412ecbb87 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -1,7 +1,7 @@ import { GridClientErrors, ValidationError } from "@threefold/types"; import * as PATH from "path"; -import { RMB } from "../clients"; +import { GqlNodeContract, RMB } from "../clients"; import { TFClient } from "../clients/tf-grid/client"; import { GridClientConfig } from "../config"; import { formatErrorMessage } from "../helpers"; @@ -22,6 +22,13 @@ import { Workload, WorkloadTypes } from "../zos/workload"; import { Zmachine, ZmachineResult } from "../zos/zmachine"; import { Zmount } from "../zos/zmount"; +const modulesNames = { + machines: "vm", + kubernetes: "kubernetes", + gateways: "gateway", + qsfs_zdbs: "QSFS", + zdb: "zdb", +}; class BaseModule { moduleName = ""; projectName = ""; @@ -31,6 +38,7 @@ class BaseModule { twinDeploymentHandler: TwinDeploymentHandler; backendStorage: BackendStorage; tfClient: TFClient; + contracts: GqlNodeContract[]; constructor(public config: GridClientConfig) { this.projectName = config.projectName; @@ -57,7 +65,7 @@ class BaseModule { return PATH.join(this.config.storePath, this.projectName, this.moduleName, name); } - async getDeploymentContracts(name: string) { + async getOldDeploymentContracts(name: string) { const path = this.getNewDeploymentPath(name, "contracts.json"); const contracts = await this.backendStorage.load(path); if (!contracts) { @@ -66,22 +74,25 @@ class BaseModule { return contracts; } + async getDeploymentContracts(name: string) { + const contracts = await this.getMyContracts(); + const filteredContracts = contracts.filter(c => { + const deploymentData = JSON.parse(c.deploymentData) as { name: string }; + return deploymentData.name === name; + }); + if (!filteredContracts) { + return []; + } + return filteredContracts; + } + async save(name: string, contracts: Record) { const contractsPath = PATH.join(this.getNewDeploymentPath(name), "contracts.json"); const wireguardPath = PATH.join(this.getDeploymentPath(name), `${name}.conf`); - const oldContracts = await this.getDeploymentContracts(name); + const oldContracts = await this.getOldDeploymentContracts(name); let StoreContracts = oldContracts; let backendOperations = []; - for (const contract of contracts["created"]) { - StoreContracts.push({ - contract_id: contract["contractId"], - node_id: contract["contractType"]["nodeContract"]["nodeId"], - }); - const contractPath = PATH.join(this.config.storePath, "contracts", `${contract["contractId"]}.json`); - const contractInfo = { projectName: this.projectName, moduleName: this.moduleName, deploymentName: name }; - backendOperations = backendOperations.concat(await this.backendStorage.dump(contractPath, contractInfo)); - } for (const contract of contracts["deleted"]) { StoreContracts = StoreContracts.filter(c => c["contract_id"] !== contract["contractId"]); const contractPath = PATH.join(this.config.storePath, "contracts", `${contract["contractId"]}.json`); @@ -147,9 +158,30 @@ class BaseModule { await Promise.all(keys.map(__updatePath).flat(1)); } + private async getMyContracts(fetch = false) { + if (fetch || !this.contracts) { + this.contracts = await this.tfClient.contracts.listMyNodeContracts({ graphqlURL: this.config.graphqlURL }); + } + return this.contracts.filter(c => { + const deploymentData = JSON.parse(c.deploymentData) as { type: string; name: string; projectName: string }; + if ( + (this.projectName === "" && deploymentData.projectName === this.projectName) || + (this.projectName !== "" && deploymentData.projectName.startsWith(this.projectName)) + ) + if (deploymentData.type === modulesNames[this.moduleName]) return true; + return false; + }); + } + async _list(): Promise { await this._migrateListKeys(); - return this.backendStorage.list(this.getNewDeploymentPath("")); + //TODO: optimize it by doing the filtering directly on graphql + const contracts = await this.getMyContracts(true); + return contracts.map(c => { + const deploymentData = JSON.parse(c.deploymentData) as { name: string }; + console.log(this.projectName, deploymentData.name); + return deploymentData.name; + }); } async exists(name: string): Promise { @@ -166,10 +198,10 @@ class BaseModule { } async _getDeploymentNodeIds(name: string): Promise { - const nodeIds = []; + const nodeIds: number[] = []; const contracts = await this.getDeploymentContracts(name); for (const contract of contracts) { - nodeIds.push(contract["node_id"]); + nodeIds.push(contract.nodeID); } return nodeIds; } @@ -177,18 +209,20 @@ class BaseModule { async _getContractIdFromNodeId(name: string, nodeId: number): Promise { const contracts = await this.getDeploymentContracts(name); for (const contract of contracts) { - if (contract["node_id"] === nodeId) { - return contract["contract_id"]; + if (contract.nodeID === nodeId) { + return +contract.contractID; } } + return 0; } async _getNodeIdFromContractId(name: string, contractId: number): Promise { const contracts = await this.getDeploymentContracts(name); for (const contract of contracts) { - if (contract["contract_id"] === contractId) { - return contract["node_id"]; + if (+contract.contractID === contractId) { + return contract.nodeID; } } + return 0; } async _getWorkloadsByTypes(deploymentName: string, deployments, types: WorkloadTypes[]): Promise { @@ -294,14 +328,14 @@ class BaseModule { } await this.tfClient.connect(); for (const contract of contracts) { - const c = await this.tfClient.contracts.get({ id: contract["contract_id"] }); + const c = await this.tfClient.contracts.get({ id: +contract.contractID }); if (c === null) { - await this.save(name, { created: [], deleted: [{ contractId: contract["contract_id"] }] }); + await this.save(name, { created: [], deleted: [{ contractId: +contract.contractID }] }); continue; } const nodes = new Nodes(this.config.graphqlURL, this.config.proxyURL, this.config.rmbClient); - const node_twin_id = await nodes.getNodeTwinId(contract["node_id"]); - const payload = JSON.stringify({ contract_id: contract["contract_id"] }); + const node_twin_id = await nodes.getNodeTwinId(contract.nodeID); + const payload = JSON.stringify({ contract_id: +contract.contractID }); let deployment; try { deployment = await this.rmb.request([node_twin_id], "zos.deployment.get", payload); @@ -318,7 +352,7 @@ class BaseModule { if (found) { deployments.push(deployment); } else { - await this.save(name, { created: [], deleted: [{ contractId: contract["contract_id"] }] }); + await this.save(name, { created: [], deleted: [{ contractId: +contract.contractID }] }); } } return deployments; @@ -569,7 +603,6 @@ class BaseModule { } finalTwinDeployments.push(twinDeployment); const contracts = await this.twinDeploymentHandler.handle(finalTwinDeployments); - await this.save(deployment_name, contracts); return { contracts: contracts }; } diff --git a/packages/grid_client/src/modules/zdb.ts b/packages/grid_client/src/modules/zdb.ts index 260fce306a..1df82ec4a1 100644 --- a/packages/grid_client/src/modules/zdb.ts +++ b/packages/grid_client/src/modules/zdb.ts @@ -13,7 +13,7 @@ import { AddZDBModel, DeleteZDBModel, ZDBDeleteModel, ZDBGetModel, ZDBSModel } f import { checkBalance } from "./utils"; class ZdbsModule extends BaseModule { - fileName = "zdbs.json"; + moduleName = "zdb"; workloadTypes = [WorkloadTypes.zdb]; zdb: ZdbHL; constructor(public config: GridClientConfig) { diff --git a/packages/playground/src/components/vm_deployment_table.vue b/packages/playground/src/components/vm_deployment_table.vue index 952b5f2a44..276769fa66 100644 --- a/packages/playground/src/components/vm_deployment_table.vue +++ b/packages/playground/src/components/vm_deployment_table.vue @@ -160,19 +160,15 @@ async function loadDeployments() { await migrateModule(grid!.gateway); } - const filter = - props.projectName.toLowerCase() === ProjectName.VM.toLowerCase() - ? undefined - : ([vm]: [{ flist: string }]) => vm.flist.replace(/-/g, "").includes(props.projectName.toLowerCase()); - const chunk3 = - props.projectName.toLowerCase() === ProjectName.Fullvm.toLowerCase() - ? { count: 0, items: [] } - : await loadVms(updateGrid(grid!, { projectName: "" }), { filter }); + props.projectName.toLowerCase() === ProjectName.VM.toLowerCase() + ? await loadVms(updateGrid(grid!, { projectName: "" })) + : { count: 0, items: [] }; if (chunk3.count > 0 && migrateGateways) { await migrateModule(grid!.gateway); } + console.log({ chunk1, chunk2, chunk3 }); const vms = mergeLoadedDeployments(chunk1, chunk2, chunk3 as any); failedDeployments.value = [ ...(Array.isArray((chunk1 as any).failedDeployments) ? (chunk1 as any).failedDeployments : []), From ac402957e5936a215c639aa1a98dfa62655a4ca1 Mon Sep 17 00:00:00 2001 From: ahmedhanafy725 Date: Sun, 4 Feb 2024 18:59:30 +0200 Subject: [PATCH 02/42] Update filtering the contracts while listing the deployments --- packages/grid_client/src/modules/base.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index f412ecbb87..6f3de5e11c 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -164,12 +164,9 @@ class BaseModule { } return this.contracts.filter(c => { const deploymentData = JSON.parse(c.deploymentData) as { type: string; name: string; projectName: string }; - if ( - (this.projectName === "" && deploymentData.projectName === this.projectName) || - (this.projectName !== "" && deploymentData.projectName.startsWith(this.projectName)) - ) - if (deploymentData.type === modulesNames[this.moduleName]) return true; - return false; + return ( + deploymentData.projectName.startsWith(this.projectName) && deploymentData.type === modulesNames[this.moduleName] + ); }); } @@ -179,7 +176,6 @@ class BaseModule { const contracts = await this.getMyContracts(true); return contracts.map(c => { const deploymentData = JSON.parse(c.deploymentData) as { name: string }; - console.log(this.projectName, deploymentData.name); return deploymentData.name; }); } From 8b5beb8ee9d20bae0c69ac4614fb07bd54ec9286 Mon Sep 17 00:00:00 2001 From: ahmedhanafy725 Date: Mon, 5 Feb 2024 14:19:47 +0200 Subject: [PATCH 03/42] Update the contract field on the failed deployment to the new naming --- packages/playground/src/components/k8s_deployment_table.vue | 4 ++-- packages/playground/src/components/vm_deployment_table.vue | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/playground/src/components/k8s_deployment_table.vue b/packages/playground/src/components/k8s_deployment_table.vue index 2f6f5cfc99..ee4a5630af 100644 --- a/packages/playground/src/components/k8s_deployment_table.vue +++ b/packages/playground/src/components/k8s_deployment_table.vue @@ -38,8 +38,8 @@ }} diff --git a/packages/playground/src/components/vm_deployment_table.vue b/packages/playground/src/components/vm_deployment_table.vue index 276769fa66..36dd942473 100644 --- a/packages/playground/src/components/vm_deployment_table.vue +++ b/packages/playground/src/components/vm_deployment_table.vue @@ -138,7 +138,7 @@ const failedDeployments = ref< { name: string; nodes?: number[]; - contracts?: { contract_id: number; node_id: number }[]; + contracts?: { contractID: number; node_id: number }[]; }[] >([]); @@ -245,7 +245,7 @@ const failedDeploymentList = computed(() => { const contractMessage = contracts.length > 0 ? ` With Contract ID: ${contracts - .map(c => c.contract_id) + .map(c => c.contractID) .join(", ")}.` : ""; From b4377a56073821949d62da8cd30782ba29f4ab6d Mon Sep 17 00:00:00 2001 From: ahmedhanafy725 Date: Mon, 5 Feb 2024 14:29:30 +0200 Subject: [PATCH 04/42] Remove saving the deployment data after deploying the workload from kvstore --- packages/grid_client/src/modules/gateway.ts | 2 -- packages/grid_client/src/modules/k8s.ts | 1 - packages/grid_client/src/modules/machines.ts | 1 - packages/grid_client/src/modules/qsfs_zdbs.ts | 1 - packages/grid_client/src/modules/zdb.ts | 1 - 5 files changed, 6 deletions(-) diff --git a/packages/grid_client/src/modules/gateway.ts b/packages/grid_client/src/modules/gateway.ts index f9e2a00cf8..f3c009ff89 100644 --- a/packages/grid_client/src/modules/gateway.ts +++ b/packages/grid_client/src/modules/gateway.ts @@ -53,7 +53,6 @@ class GWModule extends BaseModule { options.solutionProviderId, ); const contracts = await this.twinDeploymentHandler.handle(twinDeployments); - await this.save(options.name, contracts); return { contracts: contracts }; } @@ -82,7 +81,6 @@ class GWModule extends BaseModule { options.solutionProviderId, ); const contracts = await this.twinDeploymentHandler.handle(twinDeployments); - await this.save(options.name, contracts); return { contracts: contracts }; } diff --git a/packages/grid_client/src/modules/k8s.ts b/packages/grid_client/src/modules/k8s.ts index 988a290319..13cd8d0838 100644 --- a/packages/grid_client/src/modules/k8s.ts +++ b/packages/grid_client/src/modules/k8s.ts @@ -183,7 +183,6 @@ class K8sModule extends BaseModule { events.emit("logs", `Start creating the cluster with name ${options.name}`); const [deployments, , wireguardConfig] = await this._createDeployment(options); const contracts = await this.twinDeploymentHandler.handle(deployments); - await this.save(options.name, contracts); return { contracts: contracts, wireguard_config: wireguardConfig }; } diff --git a/packages/grid_client/src/modules/machines.ts b/packages/grid_client/src/modules/machines.ts index f343aeb85c..2662ffa212 100644 --- a/packages/grid_client/src/modules/machines.ts +++ b/packages/grid_client/src/modules/machines.ts @@ -92,7 +92,6 @@ class MachinesModule extends BaseModule { events.emit("logs", `Start creating the machine deployment with name ${options.name}`); const [twinDeployments, , wireguardConfig] = await this._createDeployment(options); const contracts = await this.twinDeploymentHandler.handle(twinDeployments); - await this.save(options.name, contracts); return { contracts: contracts, wireguard_config: wireguardConfig }; } diff --git a/packages/grid_client/src/modules/qsfs_zdbs.ts b/packages/grid_client/src/modules/qsfs_zdbs.ts index 4a1dab5354..b0904a0dd6 100644 --- a/packages/grid_client/src/modules/qsfs_zdbs.ts +++ b/packages/grid_client/src/modules/qsfs_zdbs.ts @@ -66,7 +66,6 @@ class QSFSZdbsModule extends BaseModule { events.emit("logs", `Start creating the QSFS ZDBs deployment with name ${options.name}`); const twinDeployments = await this._createDeployment(options); const contracts = await this.twinDeploymentHandler.handle(twinDeployments); - await this.save(options.name, contracts); return { contracts: contracts }; } diff --git a/packages/grid_client/src/modules/zdb.ts b/packages/grid_client/src/modules/zdb.ts index 1df82ec4a1..cc445e2c15 100644 --- a/packages/grid_client/src/modules/zdb.ts +++ b/packages/grid_client/src/modules/zdb.ts @@ -60,7 +60,6 @@ class ZdbsModule extends BaseModule { events.emit("logs", `Start creating the ZDB deployment with name ${options.name}`); const twinDeployments = await this._createDeployment(options); const contracts = await this.twinDeploymentHandler.handle(twinDeployments); - await this.save(options.name, contracts); return { contracts: contracts }; } From 6be3dae87751de7f080e3bd9cc6660d1795633cf Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Tue, 6 Feb 2024 14:33:35 +0200 Subject: [PATCH 05/42] feat: Add markAsFromAnotherClient helper --- packages/playground/src/utils/helpers.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/playground/src/utils/helpers.ts b/packages/playground/src/utils/helpers.ts index 367388570d..b44d036627 100644 --- a/packages/playground/src/utils/helpers.ts +++ b/packages/playground/src/utils/helpers.ts @@ -75,3 +75,14 @@ export function toGigaBytes(value?: number) { const gb = value / giga; return parseFloat(gb.toFixed(2)); } + +export function markAsFromAnotherClient(deployment: any): any { + deployment.fromAnotherClient = true; + if (Array.isArray(deployment)) { + deployment.map(t => { + t.fromAnotherClient = true; + return t; + }); + } + return deployment; +} From ab635059bc0b8be2a2543e221e2752aab6b1ac45 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Tue, 6 Feb 2024 14:35:51 +0200 Subject: [PATCH 06/42] feat: use markAsFromAnotherClient and add switch to toggle between show/hide all deployments in vm_deployments_list --- .../src/components/vm_deployment_table.vue | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/playground/src/components/vm_deployment_table.vue b/packages/playground/src/components/vm_deployment_table.vue index 5eab8826ea..03e6f2eb18 100644 --- a/packages/playground/src/components/vm_deployment_table.vue +++ b/packages/playground/src/components/vm_deployment_table.vue @@ -49,9 +49,17 @@ + + (); const items = ref([]); const showDialog = ref(false); const showEncryption = ref(false); +const showAllDeployments = ref(false); const failedDeployments = ref< { name: string; @@ -170,10 +180,13 @@ async function loadDeployments() { props.projectName.toLowerCase() === ProjectName.VM.toLowerCase() ? await loadVms(updateGrid(grid!, { projectName: "" })) : { count: 0, items: [] }; + if (chunk3.count > 0 && migrateGateways) { await migrateModule(grid!.gateway); } + chunk3.items = chunk3.items.map(markAsFromAnotherClient); + console.log({ chunk1, chunk2, chunk3 }); const vms = mergeLoadedDeployments(chunk1, chunk2, chunk3 as any); failedDeployments.value = [ @@ -182,6 +195,8 @@ async function loadDeployments() { ...(Array.isArray((chunk3 as any).failedDeployments) ? (chunk3 as any).failedDeployments : []), ]; + console.log({ vms }); + count.value = vms.count; items.value = vms.items.map(([leader, ...workers]) => { if (workers.length) { From 3c6b9eb27ccee4dde2938ead1f090a3b7bd056fd Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Tue, 6 Feb 2024 14:36:03 +0200 Subject: [PATCH 07/42] feat: use markAsFromAnotherClient and add switch to toggle between show/hide all deployments in k8s_deployments_list --- .../playground/src/components/k8s_deployment_table.vue | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/playground/src/components/k8s_deployment_table.vue b/packages/playground/src/components/k8s_deployment_table.vue index 4aef3dec64..e31691b764 100644 --- a/packages/playground/src/components/k8s_deployment_table.vue +++ b/packages/playground/src/components/k8s_deployment_table.vue @@ -59,6 +59,8 @@ + + ([]); const props = defineProps<{ @@ -127,7 +131,6 @@ defineEmits<{ (event: "update:model-value", value: any[]): void }>(); const count = ref(); const items = ref([]); const loading = ref(false); -const namesOfFailedDeployments = ref(""); onMounted(loadDeployments); async function loadDeployments() { @@ -138,6 +141,8 @@ async function loadDeployments() { const chunk2 = await loadK8s(updateGrid(grid!, { projectName: props.projectName.toLowerCase() })); const chunk3 = await loadK8s(updateGrid(grid!, { projectName: "" })); + chunk3.items = chunk3.items.map(markAsFromAnotherClient); + const clusters = mergeLoadedDeployments(chunk1, chunk2, chunk3); failedDeployments.value = [ ...(Array.isArray((chunk1 as any).failedDeployments) ? (chunk1 as any).failedDeployments : []), From 70df69f311cbabb1866aceb47c9fc1c856f8c5f5 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Tue, 6 Feb 2024 15:36:37 +0200 Subject: [PATCH 08/42] - feat: Support disable in icon_action_btn - feat: Disable manage_domains for vms from another client --- packages/playground/src/components/icon_action_btn.vue | 2 ++ packages/playground/src/weblets/tf_deployment_list.vue | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/playground/src/components/icon_action_btn.vue b/packages/playground/src/components/icon_action_btn.vue index 857bf748c8..e5b5cd25f9 100644 --- a/packages/playground/src/components/icon_action_btn.vue +++ b/packages/playground/src/components/icon_action_btn.vue @@ -10,6 +10,7 @@ variant="tonal" :height="height" class="my-1 mr-1" + :disabled="disabled" > {{ icon }} @@ -24,6 +25,7 @@ defineProps<{ color?: string; href?: string; height?: string; + disabled?: boolean; }>(); defineEmits(["click"]); diff --git a/packages/playground/src/weblets/tf_deployment_list.vue b/packages/playground/src/weblets/tf_deployment_list.vue index 5dcde95dd8..9d2f5bbe33 100644 --- a/packages/playground/src/weblets/tf_deployment_list.vue +++ b/packages/playground/src/weblets/tf_deployment_list.vue @@ -42,7 +42,12 @@ @click="openDialog(tabs[activeTab].value, item)" /> - + Date: Tue, 6 Feb 2024 16:17:29 +0200 Subject: [PATCH 09/42] refactor: Remove console logs --- packages/playground/src/components/vm_deployment_table.vue | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/playground/src/components/vm_deployment_table.vue b/packages/playground/src/components/vm_deployment_table.vue index 03e6f2eb18..45e0a1679a 100644 --- a/packages/playground/src/components/vm_deployment_table.vue +++ b/packages/playground/src/components/vm_deployment_table.vue @@ -187,7 +187,6 @@ async function loadDeployments() { chunk3.items = chunk3.items.map(markAsFromAnotherClient); - console.log({ chunk1, chunk2, chunk3 }); const vms = mergeLoadedDeployments(chunk1, chunk2, chunk3 as any); failedDeployments.value = [ ...(Array.isArray((chunk1 as any).failedDeployments) ? (chunk1 as any).failedDeployments : []), @@ -195,8 +194,6 @@ async function loadDeployments() { ...(Array.isArray((chunk3 as any).failedDeployments) ? (chunk3 as any).failedDeployments : []), ]; - console.log({ vms }); - count.value = vms.count; items.value = vms.items.map(([leader, ...workers]) => { if (workers.length) { From 1994bf2064aef1c5d0e9562d7702c024254d4f10 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Wed, 7 Feb 2024 14:51:10 +0200 Subject: [PATCH 10/42] fix: Elimiate a possible error due to null value of gridStore.client --- packages/playground/src/utils/nodeSelector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playground/src/utils/nodeSelector.ts b/packages/playground/src/utils/nodeSelector.ts index 99ebe69f0c..4329039230 100644 --- a/packages/playground/src/utils/nodeSelector.ts +++ b/packages/playground/src/utils/nodeSelector.ts @@ -156,7 +156,7 @@ export function normalizeNodeOptions( size: window.env.PAGE_SIZE, page: pagination.value.page, location: location || {}, - twinId: gridStore.client.twinId, + twinId: gridStore.client?.twinId, farm, }; } From 671f5ce2cf24a68714865d36e4a085f50e889c5a Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Wed, 7 Feb 2024 14:52:54 +0200 Subject: [PATCH 11/42] fix: if projectName was empty it causes a leading / which causes issue while loading k8s deployments --- packages/playground/src/utils/load_deployment.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playground/src/utils/load_deployment.ts b/packages/playground/src/utils/load_deployment.ts index c0d53369df..a7c4d2b6ec 100644 --- a/packages/playground/src/utils/load_deployment.ts +++ b/packages/playground/src/utils/load_deployment.ts @@ -151,7 +151,7 @@ export async function loadK8s(grid: GridClient) { const projectName = grid.clientOptions.projectName; const grids = (await Promise.all( - clusters.map(n => getGrid(grid.clientOptions, `${projectName}/${n}`)), + clusters.map(n => getGrid(grid.clientOptions, projectName ? `${projectName}/${n}` : n)), )) as GridClient[]; const failedDeployments: FailedDeployment[] = []; From fd0996e4d29a5383adbedf47a44b2a33e6f3efc3 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Wed, 7 Feb 2024 16:11:35 +0200 Subject: [PATCH 12/42] fix: mark only item from different client --- packages/playground/src/components/k8s_deployment_table.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playground/src/components/k8s_deployment_table.vue b/packages/playground/src/components/k8s_deployment_table.vue index e31691b764..09696a5236 100644 --- a/packages/playground/src/components/k8s_deployment_table.vue +++ b/packages/playground/src/components/k8s_deployment_table.vue @@ -141,7 +141,7 @@ async function loadDeployments() { const chunk2 = await loadK8s(updateGrid(grid!, { projectName: props.projectName.toLowerCase() })); const chunk3 = await loadK8s(updateGrid(grid!, { projectName: "" })); - chunk3.items = chunk3.items.map(markAsFromAnotherClient); + chunk3.items = chunk3.items.map(i => (!i.projectName ? markAsFromAnotherClient(i) : i)); const clusters = mergeLoadedDeployments(chunk1, chunk2, chunk3); failedDeployments.value = [ From 204f68472906a2f2db82a4204093d3052900d045 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Wed, 7 Feb 2024 16:12:13 +0200 Subject: [PATCH 13/42] feat: Move filter to graphql level instead of client level --- .../src/clients/tf-grid/contracts.ts | 27 ++++++++++++++++++- packages/grid_client/src/modules/base.ts | 15 +++++------ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/grid_client/src/clients/tf-grid/contracts.ts b/packages/grid_client/src/clients/tf-grid/contracts.ts index 8c2592129b..c9cccc39c6 100644 --- a/packages/grid_client/src/clients/tf-grid/contracts.ts +++ b/packages/grid_client/src/clients/tf-grid/contracts.ts @@ -17,6 +17,8 @@ export interface ListContractByTwinIdOptions { graphqlURL: string; twinId: number; stateList?: ContractStates[]; + type?: string; + projectName?: string; } export interface ContractUsedResources { @@ -81,6 +83,8 @@ export interface ListContractByAddressOptions { export interface ListMyContractOptions { graphqlURL: string; stateList?: ContractStates[]; + type?: string; + projectName?: string; } export interface GetConsumptionOptions { @@ -146,10 +150,29 @@ class TFContracts extends Contracts { const state = `[${options.stateList.join(", ")}]`; const gqlClient = new Graphql(options.graphqlURL); const opts = `(where: {twinID_eq: ${options.twinId}, state_in: ${state}}, orderBy: twinID_ASC)`; + + // filter contracts based on deploymentData + let filterQuery = ""; + if (options.type || options.projectName) { + filterQuery = " , AND: ["; + + if (options.type) { + // eslint-disable-next-line no-useless-escape + filterQuery += `{ deploymentData_contains: \"\\\"type\\\":\\\"${options.type}\\\"\" },`; + } + + if (options.projectName) { + // eslint-disable-next-line no-useless-escape + filterQuery += `{ deploymentData_contains: \"\\\"projectName\\\":\\\"${options.projectName}\" }`; + } + + filterQuery += "]"; + } + try { const nodeContractsCount = await gqlClient.getItemTotalCount("nodeContracts", opts); const body = `query getContracts($nodeContractsCount: Int!){ - nodeContracts(where: {twinID_eq: ${options.twinId}, state_in: ${state}}, limit: $nodeContractsCount) { + nodeContracts(where: {twinID_eq: ${options.twinId}, state_in: ${state}${filterQuery}}, limit: $nodeContractsCount) { contractID deploymentData state @@ -255,6 +278,8 @@ class TFContracts extends Contracts { graphqlURL: options.graphqlURL, twinId: twinId, stateList: options.stateList, + type: options.type, + projectName: options.projectName, }); } diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index 6f3de5e11c..f7fb47905d 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -160,19 +160,18 @@ class BaseModule { private async getMyContracts(fetch = false) { if (fetch || !this.contracts) { - this.contracts = await this.tfClient.contracts.listMyNodeContracts({ graphqlURL: this.config.graphqlURL }); + this.contracts = await this.tfClient.contracts.listMyNodeContracts({ + graphqlURL: this.config.graphqlURL, + type: modulesNames[this.moduleName], + projectName: this.projectName, + }); } - return this.contracts.filter(c => { - const deploymentData = JSON.parse(c.deploymentData) as { type: string; name: string; projectName: string }; - return ( - deploymentData.projectName.startsWith(this.projectName) && deploymentData.type === modulesNames[this.moduleName] - ); - }); + + return this.contracts; } async _list(): Promise { await this._migrateListKeys(); - //TODO: optimize it by doing the filtering directly on graphql const contracts = await this.getMyContracts(true); return contracts.map(c => { const deploymentData = JSON.parse(c.deploymentData) as { name: string }; From ef75fac9a5b3832c970700973dd4830cdcb1d736 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Wed, 7 Feb 2024 16:54:06 +0200 Subject: [PATCH 14/42] feat: Store parsed value instead of parsing deploymentData over and over --- .../src/clients/tf-grid/contracts.ts | 1 + packages/grid_client/src/modules/base.ts | 31 ++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/grid_client/src/clients/tf-grid/contracts.ts b/packages/grid_client/src/clients/tf-grid/contracts.ts index c9cccc39c6..52628246c8 100644 --- a/packages/grid_client/src/clients/tf-grid/contracts.ts +++ b/packages/grid_client/src/clients/tf-grid/contracts.ts @@ -49,6 +49,7 @@ export interface GqlNodeContract extends GqlBaseContract { deploymentHash: string; numberOfPublicIPs: number; resourcesUsed: ContractUsedResources; + parsedDeploymentData?: { type: string; name: string; projectName: string }; } export interface GqlRentContract extends GqlBaseContract { diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index f7fb47905d..273a468dbb 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -38,7 +38,7 @@ class BaseModule { twinDeploymentHandler: TwinDeploymentHandler; backendStorage: BackendStorage; tfClient: TFClient; - contracts: GqlNodeContract[]; + contracts: Required[]; constructor(public config: GridClientConfig) { this.projectName = config.projectName; @@ -76,14 +76,7 @@ class BaseModule { async getDeploymentContracts(name: string) { const contracts = await this.getMyContracts(); - const filteredContracts = contracts.filter(c => { - const deploymentData = JSON.parse(c.deploymentData) as { name: string }; - return deploymentData.name === name; - }); - if (!filteredContracts) { - return []; - } - return filteredContracts; + return contracts.filter(c => c.parsedDeploymentData.name === name) || []; } async save(name: string, contracts: Record) { @@ -160,11 +153,24 @@ class BaseModule { private async getMyContracts(fetch = false) { if (fetch || !this.contracts) { - this.contracts = await this.tfClient.contracts.listMyNodeContracts({ + const contracts = await this.tfClient.contracts.listMyNodeContracts({ graphqlURL: this.config.graphqlURL, type: modulesNames[this.moduleName], projectName: this.projectName, }); + + const parsedContractsName = new Set(); + const parsedContracts: Required[] = []; + + for (const contract of contracts) { + const parsedDeploymentData = JSON.parse(contract.deploymentData); + if (!parsedContractsName.has(parsedDeploymentData.name)) { + parsedContractsName.add(parsedDeploymentData.name); + parsedContracts.push({ ...contract, parsedDeploymentData }); + } + } + + this.contracts = parsedContracts; } return this.contracts; @@ -173,10 +179,7 @@ class BaseModule { async _list(): Promise { await this._migrateListKeys(); const contracts = await this.getMyContracts(true); - return contracts.map(c => { - const deploymentData = JSON.parse(c.deploymentData) as { name: string }; - return deploymentData.name; - }); + return contracts.map(c => c.parsedDeploymentData.name); } async exists(name: string): Promise { From 35183e17fd37690e64a62469e02ca34e6205553c Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Sun, 11 Feb 2024 12:43:01 +0200 Subject: [PATCH 15/42] fix: Stop filtering out workers from k8s clusters --- packages/grid_client/src/modules/base.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index 273a468dbb..6def94baef 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -76,7 +76,7 @@ class BaseModule { async getDeploymentContracts(name: string) { const contracts = await this.getMyContracts(); - return contracts.filter(c => c.parsedDeploymentData.name === name) || []; + return contracts.filter(c => c.parsedDeploymentData.name === name); } async save(name: string, contracts: Record) { @@ -159,15 +159,11 @@ class BaseModule { projectName: this.projectName, }); - const parsedContractsName = new Set(); const parsedContracts: Required[] = []; for (const contract of contracts) { const parsedDeploymentData = JSON.parse(contract.deploymentData); - if (!parsedContractsName.has(parsedDeploymentData.name)) { - parsedContractsName.add(parsedDeploymentData.name); - parsedContracts.push({ ...contract, parsedDeploymentData }); - } + parsedContracts.push({ ...contract, parsedDeploymentData }); } this.contracts = parsedContracts; @@ -176,10 +172,14 @@ class BaseModule { return this.contracts; } + private _getContractsName(contracts: Required[]): string[] { + return Array.from(new Set(contracts.map(c => c.parsedDeploymentData.name))); + } + async _list(): Promise { await this._migrateListKeys(); const contracts = await this.getMyContracts(true); - return contracts.map(c => c.parsedDeploymentData.name); + return this._getContractsName(contracts); } async exists(name: string): Promise { From 0ee9dfae697efad4bf2cd9412037b42900f567aa Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Sun, 11 Feb 2024 14:40:33 +0200 Subject: [PATCH 16/42] fix: interfaces might be empty --- packages/playground/src/components/vm_deployment_table.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playground/src/components/vm_deployment_table.vue b/packages/playground/src/components/vm_deployment_table.vue index ab4b7b496e..119e7b1e14 100644 --- a/packages/playground/src/components/vm_deployment_table.vue +++ b/packages/playground/src/components/vm_deployment_table.vue @@ -83,7 +83,7 @@