diff --git a/backend/infrahub/core/migrations/graph/m019_restore_rels_to_time.py b/backend/infrahub/core/migrations/graph/m019_restore_rels_to_time.py index 1e64b11956..caebbaa42f 100644 --- a/backend/infrahub/core/migrations/graph/m019_restore_rels_to_time.py +++ b/backend/infrahub/core/migrations/graph/m019_restore_rels_to_time.py @@ -5,7 +5,7 @@ from infrahub.core.migrations.shared import GraphMigration, MigrationResult from infrahub.log import get_logger -from ...constants import GLOBAL_BRANCH_NAME +from ...constants import GLOBAL_BRANCH_NAME, BranchSupportType from ...query import Query, QueryType if TYPE_CHECKING: @@ -28,12 +28,18 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> No query = """ MATCH (node:Node)-[global_edge:IS_RELATED {branch: $global_branch}]-(rel:Relationship) + WHERE rel.branch_support=$branch_aware MATCH (rel)-[non_global_edge:IS_RELATED]-(node_2: Node) WHERE non_global_edge.branch <> $global_branch SET global_edge.branch = non_global_edge.branch """ - params = {"global_branch": GLOBAL_BRANCH_NAME} + params = { + "global_branch": GLOBAL_BRANCH_NAME, + "branch_aware": BranchSupportType.AWARE.value, + "branch_agnostic": BranchSupportType.AGNOSTIC.value, + } + self.params.update(params) self.add_to_query(query) @@ -52,7 +58,7 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> No query = """ MATCH (node:Node)-[deleted_edge:IS_RELATED {status: "deleted"}]-(rel:Relationship) - MATCH (rel)-[active_edge:IS_RELATED {status: "active"}]-() + MATCH (rel)-[active_edge:IS_RELATED {status: "active"}]-(node) WHERE active_edge.to IS NULL AND deleted_edge.branch = active_edge.branch SET active_edge.to = deleted_edge.from """ @@ -69,61 +75,114 @@ class DeleteNodesRelsQuery(Query): async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None: """ - Some nodes may have been deleted while having corrupted state that are fixes by above migrations. + Some nodes may have been deleted while having corrupted state that are fixed by above migrations. While these nodes edges connected to Root are correctly deleted, edges connected to other `Node` through a `Relationship` node may still be active. Following query correctly deletes these edges by both setting correct to time and creating corresponding deleted edge. """ query = """ - MATCH (deleted_node: CoreStandardGroup)-[deleted_edge:IS_PART_OF {status: "deleted"}]->(:Root) + MATCH (deleted_node: Node)-[deleted_edge:IS_PART_OF {status: "deleted"}]->(:Root) MATCH (deleted_node)-[:IS_RELATED]-(rel:Relationship) + // exclude nodes having been deleted through migration. find those with same uuid and exclude the one with earlier + // timestamp on active branch + WHERE NOT EXISTS { + MATCH (deleted_node)-[e1:IS_RELATED]-(rel)-[e2:IS_RELATED]-(other_node) + WITH deleted_node, other_node, MIN(e1.from) AS min_e1_from, MIN(e2.from) AS min_e2_from + WHERE deleted_node <> other_node AND deleted_node.uuid = other_node.uuid AND min_e1_from < min_e2_from + } + // Set to time if there is an active edge on deleted edge branch - OPTIONAL MATCH (rel)-[peer_active_edge:IS_RELATED {status: "active"}]-(peer_1: Node) - WHERE peer_active_edge.branch = deleted_edge.branch AND peer_active_edge.to IS NULL - SET peer_active_edge.to = deleted_edge.from + CALL { + WITH rel, deleted_edge + OPTIONAL MATCH (rel)-[peer_active_edge {status: "active"}]-(peer_1) + WHERE peer_active_edge.branch = deleted_edge.branch AND peer_active_edge.to IS NULL + SET peer_active_edge.to = deleted_edge.from + } - // Check if deleted edge exists on this branch between Relationship and any peer_2 Node connected. Create it if it doesn't. - WITH deleted_edge.branch AS branch, deleted_edge.branch_level AS branch_level, deleted_edge.from as deleted_time, rel - MATCH (rel)-[:IS_RELATED]-(peer_2:Node) + // Get distinct rel nodes linked to a deleted node, with the time at which we should delete rel edges. + // Take the MAX time so if it does not take the deleted time of a node deleted through a duplication migration. + WITH DISTINCT rel, deleted_edge.branch AS deleted_edge_branch, + deleted_edge.branch_level AS branch_level, MAX(deleted_edge.from) as deleted_time + + MATCH (rel)-[]-(peer_2) + WHERE NOT exists((rel)-[{status: "deleted"}]-(peer_2)) + AND (rel.branch_support = $branch_agnostic + OR (rel.branch_support = $branch_aware + AND (deleted_edge_branch = $global_branch + OR exists((rel)-[{status: "active", branch: deleted_edge_branch}]-(peer_2))))) + + // Retrieve branch value on which deleted edge should be created. Note this block is only relevant when rel.branch_support = branch_aware. + // This block should always return 1 row (never 0) because there is no deleted edge at this point + // so there must be an existing active one connecting nodes. CALL { - WITH rel, peer_2, branch - OPTIONAL MATCH (rel)-[r:IS_RELATED {branch: branch}]-(peer_2) - WHERE r.status = "deleted" - RETURN r IS NOT NULL AS has_deleted_edge + WITH rel + MATCH (rel)-[active_edge {status: "active"}]-(peer_2) + RETURN active_edge.branch as active_edge_branch + LIMIT 1 } - // The branch on which `deleted` edge might be created depends on Relationship.branch_support - WITH branch, branch_level, deleted_time, rel, has_deleted_edge, peer_2 - WHERE has_deleted_edge = FALSE // only look at rel-peer_2 couples not having a deleted edge - OPTIONAL MATCH (rel)-[active_edge:IS_RELATED {status: "active"}]-(peer_3: Node) - WHERE active_edge.branch IS NOT NULL - WITH rel, active_edge, peer_3, - CASE - WHEN rel.branch_support = "agnostic" THEN $global_branch - WHEN rel.branch_support = "aware" THEN COALESCE(active_edge.branch, NULL) - ELSE NULL // Ending up here means there is no active branch between rel its peer Node, - // so there must be a deleted edge already, and thus we will not create one. - END AS branch, - branch_level, - deleted_time, - peer_2 - - // Need 2 calls to create the edge in the correct direction. Also note that MERGE ensures we do not create multiple times. + + WITH DISTINCT + CASE + // Branch on which `deleted` edge should be created depends on rel.branch_support. + WHEN rel.branch_support = $branch_agnostic + THEN $global_branch + ELSE active_edge_branch + END AS branch, + branch_level, + deleted_time, + peer_2, + rel + + // Then creates `deleted` edge. + // Below CALL subqueries are called once for each rel-peer_2 pair for which we want to create a deleted edge. + // Note that with current infrahub relationships edges design, only one of this CALL should be matched per pair. + CALL { WITH rel, peer_2, branch, branch_level, deleted_time MATCH (rel)-[:IS_RELATED]->(peer_2) MERGE (rel)-[:IS_RELATED {status: "deleted", branch: branch, branch_level: branch_level, from: deleted_time}]->(peer_2) } + + CALL { + WITH rel, peer_2, branch, branch_level, deleted_time + MATCH (rel)-[:IS_PROTECTED]->(peer_2) + MERGE (rel)-[:IS_PROTECTED {status: "deleted", branch: branch, branch_level: branch_level, from: deleted_time}]->(peer_2) + } + + CALL { + WITH rel, peer_2, branch, branch_level, deleted_time + MATCH (rel)-[:IS_VISIBLE]->(peer_2) + MERGE (rel)-[:IS_VISIBLE {status: "deleted", branch: branch, branch_level: branch_level, from: deleted_time}]->(peer_2) + } + CALL { WITH rel, peer_2, branch, branch_level, deleted_time MATCH (rel)<-[:IS_RELATED]-(peer_2) MERGE (rel)<-[:IS_RELATED {status: "deleted", branch: branch, branch_level: branch_level, from: deleted_time}]-(peer_2) } + + CALL { + WITH rel, peer_2, branch, branch_level, deleted_time + MATCH (rel)<-[:IS_PROTECTED]-(peer_2) + MERGE (rel)<-[:IS_PROTECTED {status: "deleted", branch: branch, branch_level: branch_level, from: deleted_time}]-(peer_2) + } + + CALL { + WITH rel, peer_2, branch, branch_level, deleted_time + MATCH (rel)<-[:IS_VISIBLE]-(peer_2) + MERGE (rel)<-[:IS_VISIBLE {status: "deleted", branch: branch, branch_level: branch_level, from: deleted_time}]-(peer_2) + } """ - params = {"global_branch": GLOBAL_BRANCH_NAME} + params = { + "global_branch": GLOBAL_BRANCH_NAME, + "branch_aware": BranchSupportType.AWARE.value, + "branch_agnostic": BranchSupportType.AGNOSTIC.value, + } + self.params.update(params) self.add_to_query(query) @@ -133,8 +192,8 @@ class Migration019(GraphMigration): Fix corrupted state introduced by Migration012 when duplicating a CoreAccount (branch Aware) being part of a CoreStandardGroup (branch Agnostic). Database is corrupted at multiple points: - Old CoreAccount node <> group_member node `active` edge has no `to` time (possibly because of #5590). - - Old CoreAccount node <> group_member node `deleted` edge is on `-global-` branch instead of `main`. - - New CoreAccount node <> group_member node `active` edge is on `-global-` branch instead of `main`. + - Old CoreAccount node <> group_member node `deleted` edge is on `$global_branch` branch instead of `main`. + - New CoreAccount node <> group_member node `active` edge is on `$global_branch` branch instead of `main`. Also, users having deleted corresponding CoreStandardGroup will also have the following data corruption, as deletion did not happen correctly due to above issues: